diff --git a/.github/workflows/deno-deno-cargo-product.yml b/.github/workflows/deno-deno-cargo-product.yml index 4d9bf872a..d0e70ca4b 100644 --- a/.github/workflows/deno-deno-cargo-product.yml +++ b/.github/workflows/deno-deno-cargo-product.yml @@ -1,7 +1,6 @@ --- name: 'deno: Deno Cargo product run' -run-name: "${{ format('Deno Cargo fresh | {0} | CLI {1}', inputs.cache_profile || - 'cargo-product', inputs.cli_version || 'Action default') }}" +run-name: "Deno Cargo / ${{ inputs.cache_profile || 'cargo-product' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/deno-deno-cargo-rolling-chain.yml b/.github/workflows/deno-deno-cargo-rolling-chain.yml index 192cb0357..9197228ec 100644 --- a/.github/workflows/deno-deno-cargo-rolling-chain.yml +++ b/.github/workflows/deno-deno-cargo-rolling-chain.yml @@ -1,8 +1,6 @@ --- name: 'deno: Deno Cargo rolling run' -run-name: "${{ github.event_name == 'push' && 'Deno Cargo rolling pinned source update' - || format('Deno Cargo rolling {0} | {1} to {2}', inputs.cache_profile || 'cargo-product', - inputs.base_sha, inputs.head_sha) }}" +run-name: "Deno Cargo / ${{ inputs.cache_profile || 'cargo-product' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.base_sha }} to ${{ inputs.head_sha }}" permissions: actions: read contents: read diff --git a/.github/workflows/grpc-grpc-bazel-benchmark.yml b/.github/workflows/grpc-grpc-bazel-benchmark.yml index 0e30aa07d..c1b94da69 100644 --- a/.github/workflows/grpc-grpc-bazel-benchmark.yml +++ b/.github/workflows/grpc-grpc-bazel-benchmark.yml @@ -1,8 +1,6 @@ --- name: 'grpc: gRPC Bazel Benchmark' -run-name: "${{ format('Benchmark {0} | {1}', inputs.cli_version != '' && 'Canary' - || 'Rolling', github.event_name == 'push' && 'Upstream update' || format('Manual - dispatch ({0})', github.ref_name)) }}" +run-name: "gRPC Bazel | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml b/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml index 41ec801dc..12972f3a6 100644 --- a/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml +++ b/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml @@ -1,10 +1,6 @@ --- name: 'grpc: gRPC Bazel Fresh' -run-name: "${{ format('Benchmark {0} | {1}', github.event_name == 'pull_request' && - 'Validation' || inputs.cli_version != '' && 'Canary' || 'Fresh', github.event_name - == 'schedule' && 'Weekly' || github.event_name == 'pull_request' && format('PR #{0}', - github.event.pull_request.number) || format('Manual dispatch ({0})', github.ref_name)) - }}" +run-name: "gRPC Bazel | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/native-fresh-benchmark.yml b/.github/workflows/native-fresh-benchmark.yml index 0ba1421f2..6de0ed6d6 100644 --- a/.github/workflows/native-fresh-benchmark.yml +++ b/.github/workflows/native-fresh-benchmark.yml @@ -1,7 +1,6 @@ --- name: Native comparison fresh -run-name: "${{ inputs.case_id }} ${{ inputs.variant }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }}${{ inputs.variant != '' && format(' / {0}', inputs.variant) || '' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/native-rolling-benchmark.yml b/.github/workflows/native-rolling-benchmark.yml index 07eadad28..734ae269c 100644 --- a/.github/workflows/native-rolling-benchmark.yml +++ b/.github/workflows/native-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: Native comparison rolling -run-name: "${{ inputs.case_id }} ${{ inputs.variant }} rolling ${{ inputs.series_id - }} sample ${{ inputs.sample }}" +run-name: "${{ inputs.case_id }}${{ inputs.variant != '' && format(' / {0}', inputs.variant) || '' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/nix-fresh-benchmark.yml b/.github/workflows/nix-fresh-benchmark.yml index 0dacd26dc..ae94d13d8 100644 --- a/.github/workflows/nix-fresh-benchmark.yml +++ b/.github/workflows/nix-fresh-benchmark.yml @@ -1,7 +1,6 @@ --- name: Nix comparison fresh -run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/nix-rolling-benchmark.yml b/.github/workflows/nix-rolling-benchmark.yml index 59bce02d4..f684684cd 100644 --- a/.github/workflows/nix-rolling-benchmark.yml +++ b/.github/workflows/nix-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: Nix comparison rolling -run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/obs-rolling-benchmark.yml b/.github/workflows/obs-rolling-benchmark.yml index cb84978e9..32cf94945 100644 --- a/.github/workflows/obs-rolling-benchmark.yml +++ b/.github/workflows/obs-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: OBS Studio comparison rolling -run-name: obs-studio ${{ inputs.cache_tool }} rolling ${{ inputs.series_id }} sample - ${{ inputs.sample }} +run-name: "obs-studio / ${{ inputs.cache_tool || 'all' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" 'on': workflow_dispatch: inputs: diff --git a/.github/workflows/obs-studio-obs-actions-cache.yml b/.github/workflows/obs-studio-obs-actions-cache.yml index f86ba64ca..2233dbccc 100644 --- a/.github/workflows/obs-studio-obs-actions-cache.yml +++ b/.github/workflows/obs-studio-obs-actions-cache.yml @@ -1,9 +1,6 @@ --- name: 'obs-studio: OBS Studio Actions Cache benchmark' -run-name: "${{ github.event_name == 'pull_request' && format('OBS validation | PR - #{0}', github.event.pull_request.number) || github.event_name == 'push' && 'OBS - rolling | Upstream update' || format('OBS Actions Cache | Manual dispatch ({0})', - github.ref_name) }}" +run-name: "obs-studio / ${{ inputs.cache_tool || 'all' }} / Actions Cache | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/obs-studio-obs-boringcache.yml b/.github/workflows/obs-studio-obs-boringcache.yml index 7796b7f01..423384c5a 100644 --- a/.github/workflows/obs-studio-obs-boringcache.yml +++ b/.github/workflows/obs-studio-obs-boringcache.yml @@ -1,10 +1,6 @@ --- name: 'obs-studio: OBS Studio BoringCache benchmark' -run-name: "${{ github.event_name == 'pull_request' && format('OBS validation | PR - #{0}', github.event.pull_request.number) || inputs.cli_version != '' && format('OBS - canary | CLI {0}', inputs.cli_version) || github.event_name == 'push' && 'OBS rolling - | Upstream update' || format('OBS BoringCache | Manual dispatch ({0})', github.ref_name) - }}" +run-name: "obs-studio / ${{ inputs.cache_tool || 'all' }} / BoringCache | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/reapi-fresh-benchmark.yml b/.github/workflows/reapi-fresh-benchmark.yml index b1b322b74..3ffa84c1c 100644 --- a/.github/workflows/reapi-fresh-benchmark.yml +++ b/.github/workflows/reapi-fresh-benchmark.yml @@ -1,7 +1,6 @@ --- name: REAPI comparison fresh -run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/reapi-rolling-benchmark.yml b/.github/workflows/reapi-rolling-benchmark.yml index 178ca1d4a..11149202d 100644 --- a/.github/workflows/reapi-rolling-benchmark.yml +++ b/.github/workflows/reapi-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: REAPI comparison rolling -run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/source-case.yml b/.github/workflows/source-case.yml index bf9cef333..e40b525e3 100644 --- a/.github/workflows/source-case.yml +++ b/.github/workflows/source-case.yml @@ -10,7 +10,7 @@ permissions: actions: read jobs: previous: - name: '${{ inputs.case_id }} previous rolling run' + name: Reconcile previous runs runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -48,7 +48,7 @@ jobs: if-no-files-found: ignore retention-days: 90 inspect: - name: '${{ inputs.case_id }} source check' + name: Check source needs: previous if: needs.previous.outputs.proceed == 'true' runs-on: ubuntu-24.04 @@ -87,7 +87,7 @@ jobs: if-no-files-found: error retention-days: 90 publish: - name: '${{ inputs.case_id }} queue rolling runs' + name: Prepare rolling runs needs: [previous, inspect] if: needs.inspect.outputs.changed == 'true' runs-on: ubuntu-24.04 diff --git a/.github/workflows/zed-zed-cargo-product.yml b/.github/workflows/zed-zed-cargo-product.yml index 8682a5cd0..117059ecb 100644 --- a/.github/workflows/zed-zed-cargo-product.yml +++ b/.github/workflows/zed-zed-cargo-product.yml @@ -1,6 +1,6 @@ --- name: 'zed: Zed Cargo layer matrix' -run-name: "${{ format('Zed Cargo layers | CLI {0}', inputs.cli_version || 'Action default') }}" +run-name: "Zed Cargo layers | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/zed-zed-cargo-rolling-auto.yml b/.github/workflows/zed-zed-cargo-rolling-auto.yml index 216471156..72e1115b6 100644 --- a/.github/workflows/zed-zed-cargo-rolling-auto.yml +++ b/.github/workflows/zed-zed-cargo-rolling-auto.yml @@ -1,7 +1,6 @@ --- name: 'zed: Zed Cargo rolling run' -run-name: "${{ format('Zed Cargo rolling {0} to {1}', inputs.base_sha, inputs.head_sha) - }}" +run-name: "Zed Cargo | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.base_sha }} to ${{ inputs.head_sha }}" permissions: actions: read checks: read diff --git a/cases/obs-studio/payload/scripts/install-ccache.sh b/cases/obs-studio/payload/scripts/install-ccache.sh index b0420e097..6356084b8 100755 --- a/cases/obs-studio/payload/scripts/install-ccache.sh +++ b/cases/obs-studio/payload/scripts/install-ccache.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -ccache_version=4.14 +ccache_version=4.14.1 ccache_archive="ccache-${ccache_version}-linux-x86_64-glibc.tar.gz" -ccache_sha256=c64760b0b85ba86068f4cd162dc42e2dc39c6f46b0cb8c1990dfccbec7a1fed0 +ccache_sha256=67f8b4cd2620b0d7bc9e834b3baea76bcdca4f2f3f47fe658c343fee58f816fa ccache_url="https://github.com/ccache/ccache/releases/download/v${ccache_version}/${ccache_archive}" storage_version=0.10 diff --git a/cases/obs-studio/payload/scripts/write_phase_result.rb b/cases/obs-studio/payload/scripts/write_phase_result.rb index 5ef340509..c6b99ae15 100755 --- a/cases/obs-studio/payload/scripts/write_phase_result.rb +++ b/cases/obs-studio/payload/scripts/write_phase_result.rb @@ -15,6 +15,7 @@ raise "Phase must be base or rolling" unless %w[base rolling].include?(phase) raise "Continuous observations require the rolling phase" if args["continuous"] && phase != "rolling" cache_hit = args["continuous"] ? args["cache_hit"] : (phase == "base" ? "false" : "true") +cache_hit = nil if cache_hit == "" raise "Cache hit must be true or false" if cache_hit && !%w[true false].include?(cache_hit) build_seconds = Float(args.fetch("build_seconds")) raise "Build time must be finite and nonnegative" unless build_seconds.finite? && build_seconds >= 0 diff --git a/docs/cadence-cutover.md b/docs/cadence-cutover.md new file mode 100644 index 000000000..ccceafe5d --- /dev/null +++ b/docs/cadence-cutover.md @@ -0,0 +1,45 @@ +# Cadence cutover + +The new workflows are merged, but central activation is off. The user deferred +activation until a compatible stable CLI is released. Historical cron triggers +remain active. Do not apply the prepared patches or set the activation variable +before the release and qualification gates in [cadence.md](cadence.md) pass. + +The [October 5 inventory](../migration/cadence-schedule-inventory-2026-10-05.json) +records 48 scheduled workflows in the 17 historical repositories represented by +the maintained suite. It records the default branch, observed head, workflow +blob SHA, cron expressions and remaining entrypoints. The seven snapshot cases +have no replaced BoringCache repository schedules. + +[Prepared patches](../migration/cadence-cutover-patches) remove only each +inventoried workflow's `schedule` event. Each edited document was parsed and +compared with the original after removing that event; manual and PR events, jobs, +permissions and other workflow behavior remain equivalent. These patches have +not been applied. Unselected repositories, including Docker corpus, are outside +this cutover. + +When release and qualification are complete: + +1. Resolve the exact stable CLI and run the full 31-target dry run on `main`. + Require the REAPI capability check to pass without omissions or substitution + of a canary. Confirm hosted checks and the retained qualification records. +2. Refresh the historical inventory. Review any changed workflow blobs before + applying their patches. Keep central activation off while removing all 48 + historical cron triggers. Preserve their manual and PR entrypoints. +3. Query every historical repository for queued, pending and running scheduled + or source-dispatched work. Allow those runs and downstream requests to finish; + retain their run IDs. Recheck that no replaced cron triggers remain. Do not + cancel unrelated runs or treat a stopped scheduler as a drained queue. +4. Set `BENCHMARK_CADENCE_ACTIVE=true` in `boringcache/benchmarks`. This transfers + source checks, weekly/nightly dispatch and canary monitoring to the central + suite. The first hourly source check should produce per-case observations; + changed cases should retain intent and requested run IDs independently. +5. Verify the first complete source cycle and the first weekly and nightly + dispatches. Compare each receipt with the 24-case / 31-fresh-target inventory, + then inspect leaf outcomes and comparison reports. Request acceptance alone + is not build completion. + +If central ownership must be stopped, unset the activation variable first and +inspect already requested central runs. Restore historical cron triggers only +after central requests have drained, so both owners do not request the same +work. Retain source receipts and observations during either transition. diff --git a/docs/cadence-qualification-2026-10-05.md b/docs/cadence-qualification-2026-10-05.md new file mode 100644 index 000000000..77978ad42 --- /dev/null +++ b/docs/cadence-qualification-2026-10-05.md @@ -0,0 +1,66 @@ +# Cadence qualification — October 5, 2026 + +All seventeen rolling targets have passed a seed run and an actual source +advancement with both providers. Zed Nix also passed both fresh samples. This is +an operational correctness review; these single-sample pairs do not establish +comparative performance. All runs use `vcli-canary-7a5b27146ebe`. + +The [machine-readable inventory](../migration/cadence-qualification-2026-10-05.json) +records frozen plans, exact sources, cache scopes and selected run IDs. Earlier +failed and cancelled cohorts remain in `results/`; corrected cohorts have new +plans. The [evidence inventory](../migration/evidence.json) links archives that +were downloaded again and verified after publication. + +Large archives use a JSON index as `bundle_url`. Its `sha256` and `bundle_bytes` +describe that index; `archive_sha256` and `archive_bytes` describe the reconstructed +tar.gz file. Download and verify each listed part, concatenate the parts in order, +verify the resulting archive, then extract it. The hosted preservation workflow +performs those checks and verifies every exported file before recording success. + +| Case / variant | Seed | Changed source | Result | +| --- | --- | --- | --- | +| immich | [37321739971](https://github.com/boringcache/benchmarks/actions/runs/37321739971) | [37323121773](https://github.com/boringcache/benchmarks/actions/runs/37323121773) | Passed | +| mastodon / server | [37321744434](https://github.com/boringcache/benchmarks/actions/runs/37321744434) | [37323126507](https://github.com/boringcache/benchmarks/actions/runs/37323126507) | Passed | +| mastodon / streaming | [37321748690](https://github.com/boringcache/benchmarks/actions/runs/37321748690) | [37323131290](https://github.com/boringcache/benchmarks/actions/runs/37323131290) | Passed | +| n8n / distroless | [37321753318](https://github.com/boringcache/benchmarks/actions/runs/37321753318) | [37323140444](https://github.com/boringcache/benchmarks/actions/runs/37323140444) | Passed | +| n8n / docker | [37321758035](https://github.com/boringcache/benchmarks/actions/runs/37321758035) | [37323145174](https://github.com/boringcache/benchmarks/actions/runs/37323145174) | Passed | +| n8n / runners | [37321762921](https://github.com/boringcache/benchmarks/actions/runs/37321762921) | [37323150431](https://github.com/boringcache/benchmarks/actions/runs/37323150431) | Passed | +| n8n / turbo | [37321767409](https://github.com/boringcache/benchmarks/actions/runs/37321767409) | [37323155106](https://github.com/boringcache/benchmarks/actions/runs/37323155106) | Passed | +| posthog / layers | [37321772493](https://github.com/boringcache/benchmarks/actions/runs/37321772493) | [37323164071](https://github.com/boringcache/benchmarks/actions/runs/37323164071) | Passed | +| executorch-buck2 | [37324032100](https://github.com/boringcache/benchmarks/actions/runs/37324032100) | [37324204556](https://github.com/boringcache/benchmarks/actions/runs/37324204556) | Passed | +| gogs-moon | [37325135541](https://github.com/boringcache/benchmarks/actions/runs/37325135541) | [37325282767](https://github.com/boringcache/benchmarks/actions/runs/37325282767) | Passed | +| opencut-moon | [37325140807](https://github.com/boringcache/benchmarks/actions/runs/37325140807) | [37325287254](https://github.com/boringcache/benchmarks/actions/runs/37325287254) | Passed | +| stackstorm-pants | [37324025714](https://github.com/boringcache/benchmarks/actions/runs/37324025714) | [37324228460](https://github.com/boringcache/benchmarks/actions/runs/37324228460) | Passed | +| msgpack-sbt | [37324036688](https://github.com/boringcache/benchmarks/actions/runs/37324036688) | [37324214890](https://github.com/boringcache/benchmarks/actions/runs/37324214890) | Passed | +| obs-studio / ccache | [37323465858](https://github.com/boringcache/benchmarks/actions/runs/37323465858) | [37324219008](https://github.com/boringcache/benchmarks/actions/runs/37324219008) | Passed | +| obs-studio / xcode | [37326133177](https://github.com/boringcache/benchmarks/actions/runs/37326133177) | [37326340221](https://github.com/boringcache/benchmarks/actions/runs/37326340221) | Passed | +| helix-nix | [37322604229](https://github.com/boringcache/benchmarks/actions/runs/37322604229) | [37323117664](https://github.com/boringcache/benchmarks/actions/runs/37323117664) | Passed | +| zed-nix | [37322636399](https://github.com/boringcache/benchmarks/actions/runs/37322636399) | [37323174641](https://github.com/boringcache/benchmarks/actions/runs/37323174641) | Passed | + +The five REAPI comparator archives identify the intended previous run and source +in `rolling-seed.json`. Both OBS comparator archives likewise identify their +intended seed in `previous-cache-source.json`. The review checks those identities +against the frozen plans; a cache flag alone is insufficient. + +The first n8n Turbo advancement retained an available BoringCache entry but +reported zero native hits. A further actual upstream revision in +[run 37325183094](https://github.com/boringcache/benchmarks/actions/runs/37325183094) +reported 70 native hits and 2 misses. Both observations remain retained. Changed +source does not guarantee a hit; absence of a hit must not be reported as reuse. + +Helix Nix passed two fresh cold/warm samples and its rolling pair. Zed Nix passed both fresh cold/warm samples and its rolling pair. Each Nix run imports one checksummed +dependency store into its provider workers and checks the dependency baseline. +The measured package is excluded from that seed. A changed package derivation +may require a build even when the rolling provider cache is available. + +The shared controller passed a 24-case hosted planning rehearsal, and fresh +dispatch passed the complete 31-target dry run. The +[publication rehearsal](../migration/rehearsals/mastodon-publication/review.md) +passed isolated conditional publication, dispatch, reconciliation and a forced +concurrent-write conflict. It used the operator token; the first activated cycle +must verify workflow-token writes on `main`. + +Central activation remains off by request until a compatible stable CLI release. +The [cutover procedure](cadence-cutover.md) preserves manual entrypoints and +drains historical runs before central ownership begins. The prepared historical +schedule patches have not been applied. diff --git a/docs/cadence.md b/docs/cadence.md index 7f891dcbb..5aafb45fd 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -1,10 +1,12 @@ # Benchmark schedules -Central activation remains off. Historical repositories still own the active +PR #42 is merged. Central activation remains off. Historical repositories still own the active weekly, canary and source schedules. Central scheduled jobs require `BENCHMARK_CADENCE_ACTIVE=true`; merging this branch does not activate them. Manual qualification remains available. Historical triggers must be retired -before ownership changes, so the migration does not duplicate runs. +before ownership changes, so the migration does not duplicate runs. The +[cutover procedure](cadence-cutover.md) includes the observed inventory and +prepared patches for 48 cron triggers in 17 repositories; none are applied yet. `suites/scheduled.json` selects the maintained suite plus Moon, Pants, Buck2, sbt and Nix: 24 source cases and 31 fresh targets. It excludes Docker corpus and @@ -71,8 +73,12 @@ keeping strict package and dependency comparisons. The earlier Zed Nix screen restored both providers' package outputs correctly but failed the dependency baseline comparison; that failed run remains retained. -Shared workflows use case, variant, lane, provider and phase labels. REAPI, Nix -and OBS rolling results use the canonical benchmark report and retain both +Run names use the case, optional variant, CLI channel and lane, followed by the +series or branch and sample. For example, `n8n / turbo | Canary fresh | main / +sample 1` and `n8n / turbo | Stable rolling | main / sample 1`. Stable tags are +not labelled as canaries merely because an exact CLI version was supplied. +Provider and phase remain visible in job labels. REAPI, Nix and OBS rolling +results use the canonical benchmark report and retain both structured evidence and the comparison summary. Reports contain measurements and verification results, without performance verdicts. @@ -83,22 +89,29 @@ The October 5 preflight rejected stable `v1.33.0`: it does not expose 31-target preflight. Weekly activation must wait for a compatible stable release; it must not substitute a canary or silently omit the REAPI cases. -The first hosted source matrix completed the cases independently and identified +The complete controller rehearsal passed all 24 cases and planned changed-source +rolling requests with the canary without publishing or dispatching builds. +The earlier source matrix completed the cases independently and identified recipe changes in Immich, Qdrant, msgpack and Zed. Their reviewed changes are recorded in [the recipe review](recipe-reviews-2026-10-05.md). New source pins and -changed recipes still require live qualification. +changed recipes are covered by the +[live qualification review](cadence-qualification-2026-10-05.md), which retains +successful, failed and pending observations separately. The repository's Actions cache limit is 200 GB, with seven-day retention. Capacity alone does not establish that a particular rolling seed was retained or restored. -Remaining activation requirements: +All seventeen rolling targets passed seed and changed-source runs. Helix and +Zed Nix each passed two fresh cold/warm samples with the common dependency seed. +Sixty qualification and rehearsal archives have been published and verified; +hosted checks and 252 local tests pass. -- Pass hosted checks for the complete controller and rolling changes. -- Qualify the new and changed rolling paths with actual source advancement, - retained seeds, verified outputs and canary CLI execution. -- Pass the corrected Zed Nix cold/warm screen using the common dependency seed. -- Verify publication and receipt reconciliation in a bounded rehearsal. +Remaining activation requirements: +- Verify main-branch workflow-token publication at cutover. The + [isolated publication rehearsal](../migration/rehearsals/mastodon-publication/review.md) + passed conditional commits, dispatch and reconciliation with the operator's + GitHub CLI; the hosted rehearsal passed inspection and dry-run publication. - Retire replaced historical cron triggers while preserving manual and release entrypoints; verify no outstanding duplicate dispatches. - After a compatible stable release, change monitoring and publication ownership diff --git a/migration/cadence-cutover-patches/benchmark-chroma.patch b/migration/cadence-cutover-patches/benchmark-chroma.patch new file mode 100644 index 000000000..ed6a0a445 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-chroma.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "2 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/chroma-fresh-benchmark.yml ++++ b/.github/workflows/chroma-fresh-benchmark.yml +@@ -25,8 +25,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 1" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "1,31 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-deno.patch b/migration/cadence-cutover-patches/benchmark-deno.patch new file mode 100644 index 000000000..895a0bf50 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-deno.patch @@ -0,0 +1,23 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "3 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "2,32 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-duckgres.patch b/migration/cadence-cutover-patches/benchmark-duckgres.patch new file mode 100644 index 000000000..a7070487a --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-duckgres.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "5 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/duckgres-fresh-benchmark.yml ++++ b/.github/workflows/duckgres-fresh-benchmark.yml +@@ -8,8 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: +- - cron: "0 3 * * 1" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "5,35 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-grpc.patch b/migration/cadence-cutover-patches/benchmark-grpc.patch new file mode 100644 index 000000000..4b2797517 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-grpc.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "7 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/grpc-bazel-fresh-benchmark.yml ++++ b/.github/workflows/grpc-bazel-fresh-benchmark.yml +@@ -7,7 +7,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 5"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/grpc-bazel-benchmark/**", ".github/workflows/grpc-bazel-benchmark.yml", ".github/workflows/grpc-bazel-fresh-benchmark.yml", "scripts/**", "upstream"] + permissions: {contents: read, actions: write} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "7,37 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-hugo-go.patch b/migration/cadence-cutover-patches/benchmark-hugo-go.patch new file mode 100644 index 000000000..4e5ba1bed --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-hugo-go.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "11 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/hugo-go-fresh-benchmark.yml ++++ b/.github/workflows/hugo-go-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 4 * * 1" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "10,40 * * * *" +- + permissions: + contents: write + actions: write diff --git a/migration/cadence-cutover-patches/benchmark-hugo.patch b/migration/cadence-cutover-patches/benchmark-hugo.patch new file mode 100644 index 000000000..43222ce92 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-hugo.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "9 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/hugo-fresh-benchmark.yml ++++ b/.github/workflows/hugo-fresh-benchmark.yml +@@ -9,7 +9,6 @@ + cli_platform: {description: "Optional BoringCache CLI platform.", required: false, type: string, default: linux-amd64} + benchmark_id_suffix: {description: "Optional suffix for an isolated comparison run.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 1"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/hugo-docker-benchmark/**", ".github/workflows/hugo-benchmark.yml", ".github/workflows/hugo-fresh-benchmark.yml", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "9,39 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-immich.patch b/migration/cadence-cutover-patches/benchmark-immich.patch new file mode 100644 index 000000000..0db25ab76 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-immich.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "12 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/immich-fresh-benchmark.yml ++++ b/.github/workflows/immich-fresh-benchmark.yml +@@ -8,7 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 2"}] + pull_request: + paths: [".gitmodules", ".github/actions/immich-docker-benchmark/**", ".github/workflows/immich-benchmark.yml", ".github/workflows/immich-fresh-benchmark.yml", "libjxl-patches/**", "scenarios/**", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "12,42 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-linkerd2.patch b/migration/cadence-cutover-patches/benchmark-linkerd2.patch new file mode 100644 index 000000000..970bf57fb --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-linkerd2.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "14 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/linkerd2-fresh-benchmark.yml ++++ b/.github/workflows/linkerd2-fresh-benchmark.yml +@@ -8,7 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 1"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/linkerd2-docker-benchmark/**", ".github/workflows/linkerd2-benchmark.yml", ".github/workflows/linkerd2-fresh-benchmark.yml", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "13,43 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-mastodon.patch b/migration/cadence-cutover-patches/benchmark-mastodon.patch new file mode 100644 index 000000000..3873ad8f3 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-mastodon.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "16 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/mastodon-fresh-benchmark.yml ++++ b/.github/workflows/mastodon-fresh-benchmark.yml +@@ -9,7 +9,6 @@ + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + upstream_ref: {description: "Optional Mastodon ref or SHA instead of the pinned submodule commit.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 4 * * 2"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/mastodon-docker-benchmark/**", ".github/workflows/mastodon-docker-benchmark.yml", ".github/workflows/mastodon-fresh-benchmark.yml", "scenarios/**", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "15,45 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-n8n.patch b/migration/cadence-cutover-patches/benchmark-n8n.patch new file mode 100644 index 000000000..5816812b8 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-n8n.patch @@ -0,0 +1,45 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "18 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/n8n-docker-fresh-benchmark.yml ++++ b/.github/workflows/n8n-docker-fresh-benchmark.yml +@@ -25,8 +25,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "30 3 * * 6" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/n8n-fresh-benchmark.yml ++++ b/.github/workflows/n8n-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 6" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "17,47 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-obs-studio.patch b/migration/cadence-cutover-patches/benchmark-obs-studio.patch new file mode 100644 index 000000000..cc403fc6b --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-obs-studio.patch @@ -0,0 +1,12 @@ +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "18,48 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch b/migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch new file mode 100644 index 000000000..edb4bcaa3 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "20 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/opentelemetry-java-gradle-fresh-benchmark.yml ++++ b/.github/workflows/opentelemetry-java-gradle-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 4" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "20,50 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-posthog.patch b/migration/cadence-cutover-patches/benchmark-posthog.patch new file mode 100644 index 000000000..1d39967e1 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-posthog.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "21 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/posthog-fresh-benchmark.yml ++++ b/.github/workflows/posthog-fresh-benchmark.yml +@@ -25,8 +25,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 3" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "21,51 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-qdrant.patch b/migration/cadence-cutover-patches/benchmark-qdrant.patch new file mode 100644 index 000000000..dcf34c5be --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-qdrant.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "23 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/qdrant-fresh-benchmark.yml ++++ b/.github/workflows/qdrant-fresh-benchmark.yml +@@ -8,7 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 1"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/qdrant-docker-benchmark/**", ".github/workflows/qdrant-benchmark.yml", ".github/workflows/qdrant-fresh-benchmark.yml", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "23,53 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-spring-ai.patch b/migration/cadence-cutover-patches/benchmark-spring-ai.patch new file mode 100644 index 000000000..11b4b3825 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-spring-ai.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "25 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/spring-ai-maven-fresh-benchmark.yml ++++ b/.github/workflows/spring-ai-maven-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 4 * * 4" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "25,55 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-storybook.patch b/migration/cadence-cutover-patches/benchmark-storybook.patch new file mode 100644 index 000000000..96b6efbb1 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-storybook.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "27 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/storybook-fresh-benchmark.yml ++++ b/.github/workflows/storybook-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 4 * * 3" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "26,56 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-zed.patch b/migration/cadence-cutover-patches/benchmark-zed.patch new file mode 100644 index 000000000..a671118c6 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-zed.patch @@ -0,0 +1,23 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "29 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "28,58 * * * *" +- + permissions: + actions: write + checks: read diff --git a/migration/cadence-qualification-2026-10-05.json b/migration/cadence-qualification-2026-10-05.json new file mode 100644 index 000000000..d59755659 --- /dev/null +++ b/migration/cadence-qualification-2026-10-05.json @@ -0,0 +1,936 @@ +{ + "schema_version": 1, + "reviewed_on": "2026-10-05", + "cli_version": "vcli-canary-7a5b27146ebe", + "cases": 12, + "targets": 17, + "activation": "deferred until a compatible stable CLI release and schedule cutover", + "observations": "All executed earlier cohorts remain retained, including failed and cancelled runs. Corrected cohorts are separate frozen plans.", + "rolling": [ + { + "case_id": "immich", + "variant": null, + "seed": { + "series": "results/immich/cadence-rolling-seed", + "run_id": 37321739971, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-immich", + "completion": "success" + }, + "advance": { + "series": "results/immich/cadence-rolling-advance", + "run_id": 37323121773, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-immich", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "mastodon", + "variant": "server", + "seed": { + "series": "results/mastodon/cadence-rolling-seed-server", + "run_id": 37321744434, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-server", + "completion": "success" + }, + "advance": { + "series": "results/mastodon/cadence-rolling-advance-server", + "run_id": 37323126507, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-server", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "mastodon", + "variant": "streaming", + "seed": { + "series": "results/mastodon/cadence-rolling-seed-streaming", + "run_id": 37321748690, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-streaming", + "completion": "success" + }, + "advance": { + "series": "results/mastodon/cadence-rolling-advance-streaming", + "run_id": 37323131290, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-streaming", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "distroless", + "seed": { + "series": "results/n8n/cadence-rolling-seed-distroless", + "run_id": 37321753318, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-distroless", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-distroless", + "run_id": 37323140444, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-distroless", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "docker", + "seed": { + "series": "results/n8n/cadence-rolling-seed-docker", + "run_id": 37321758035, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-docker", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-docker", + "run_id": 37323145174, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-docker", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "runners", + "seed": { + "series": "results/n8n/cadence-rolling-seed-runners", + "run_id": 37321762921, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-runners", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-runners", + "run_id": 37323150431, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-runners", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "turbo", + "seed": { + "series": "results/n8n/cadence-rolling-seed-turbo", + "run_id": 37321767409, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-turbo", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-turbo", + "run_id": 37323155106, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-turbo", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "additional_advancement": { + "series": "results/n8n/cadence-rolling-next-turbo", + "run_id": 37325183094, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-turbo", + "completion": "success" + }, + "reuse_observation": "The first advancement retained a usable BoringCache entry but reported zero native hits. The next actual upstream revision reported 70 native hits and 2 misses. Both observations are retained." + }, + { + "case_id": "posthog", + "variant": "layers", + "seed": { + "series": "results/posthog/cadence-rolling-seed-layers", + "run_id": 37321772493, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-posthog-layers", + "completion": "success" + }, + "advance": { + "series": "results/posthog/cadence-rolling-advance-layers", + "run_id": 37323164071, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-posthog-layers", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "executorch-buck2", + "variant": null, + "seed": { + "series": "results/executorch-buck2/cadence-rolling-seed-02", + "run_id": 37324032100, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "completion": "success" + }, + "advance": { + "series": "results/executorch-buck2/cadence-rolling-advance-02", + "run_id": 37324204556, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-executorch-buck2-cadence-20261005-executorch-buck2-v2-Linux-X64-37324032100-1", + "previous": { + "case_id": "executorch-buck2", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324032100", + "run_attempt": "1" + }, + "current": { + "case_id": "executorch-buck2", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324204556", + "run_attempt": "1" + } + } + }, + { + "case_id": "gogs-moon", + "variant": null, + "seed": { + "series": "results/gogs-moon/cadence-rolling-seed-03", + "run_id": 37325135541, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "cache_scope": "cadence-20261005-gogs-moon-v3", + "completion": "success" + }, + "advance": { + "series": "results/gogs-moon/cadence-rolling-advance-03", + "run_id": 37325282767, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "cache_scope": "cadence-20261005-gogs-moon-v3", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-gogs-moon-cadence-20261005-gogs-moon-v3-Linux-X64-37325135541-1", + "previous": { + "case_id": "gogs-moon", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "run_id": "37325135541", + "run_attempt": "1" + }, + "current": { + "case_id": "gogs-moon", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "run_id": "37325282767", + "run_attempt": "1" + } + } + }, + { + "case_id": "opencut-moon", + "variant": null, + "seed": { + "series": "results/opencut-moon/cadence-rolling-seed-03", + "run_id": 37325140807, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "cache_scope": "cadence-20261005-opencut-moon-v3", + "completion": "success" + }, + "advance": { + "series": "results/opencut-moon/cadence-rolling-advance-03", + "run_id": 37325287254, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "cache_scope": "cadence-20261005-opencut-moon-v3", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-opencut-moon-cadence-20261005-opencut-moon-v3-Linux-X64-37325140807-1", + "previous": { + "case_id": "opencut-moon", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "run_id": "37325140807", + "run_attempt": "1" + }, + "current": { + "case_id": "opencut-moon", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "run_id": "37325287254", + "run_attempt": "1" + } + } + }, + { + "case_id": "stackstorm-pants", + "variant": null, + "seed": { + "series": "results/stackstorm-pants/cadence-rolling-seed-02", + "run_id": 37324025714, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "completion": "success" + }, + "advance": { + "series": "results/stackstorm-pants/cadence-rolling-advance-02", + "run_id": 37324228460, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-stackstorm-pants-cadence-20261005-stackstorm-pants-v2-Linux-X64-37324025714-1", + "previous": { + "case_id": "stackstorm-pants", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "run_id": "37324025714", + "run_attempt": "1" + }, + "current": { + "case_id": "stackstorm-pants", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "run_id": "37324228460", + "run_attempt": "1" + } + } + }, + { + "case_id": "msgpack-sbt", + "variant": null, + "seed": { + "series": "results/msgpack-sbt/cadence-rolling-seed-02", + "run_id": 37324036688, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "completion": "success" + }, + "advance": { + "series": "results/msgpack-sbt/cadence-rolling-advance-02", + "run_id": 37324214890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-msgpack-sbt-cadence-20261005-msgpack-sbt-v2-Linux-X64-37324036688-1", + "previous": { + "case_id": "msgpack-sbt", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324036688", + "run_attempt": "1" + }, + "current": { + "case_id": "msgpack-sbt", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324214890", + "run_attempt": "1" + } + } + }, + { + "case_id": "obs-studio", + "variant": "ccache", + "seed": { + "series": "results/obs-studio/cadence-rolling-seed-ccache-02", + "run_id": 37323465858, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "completion": "success" + }, + "advance": { + "series": "results/obs-studio/cadence-rolling-advance-ccache-02", + "run_id": 37324219008, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "source_sha": "ef5814431c6af9061874470201d0b799bc84602b", + "run_id": "37323465858", + "run_attempt": "1" + } + }, + { + "case_id": "obs-studio", + "variant": "xcode", + "seed": { + "series": "results/obs-studio/cadence-rolling-seed-xcode-02", + "run_id": 37326133177, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "completion": "success" + }, + "advance": { + "series": "results/obs-studio/cadence-rolling-advance-xcode-02", + "run_id": 37326340221, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "source_sha": "ef5814431c6af9061874470201d0b799bc84602b", + "run_id": "37326133177", + "run_attempt": "1" + } + }, + { + "case_id": "helix-nix", + "variant": null, + "seed": { + "series": "results/helix-nix/cadence-rolling-seed", + "run_id": 37322604229, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "cache_scope": "cadence-20261005-helix-nix", + "completion": "success" + }, + "advance": { + "series": "results/helix-nix/cadence-rolling-advance", + "run_id": 37323117664, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "cache_scope": "cadence-20261005-helix-nix", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "zed-nix", + "variant": null, + "seed": { + "series": "results/zed-nix/cadence-rolling-seed", + "run_id": 37322636399, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "cache_scope": "cadence-20261005-zed-nix", + "completion": "success" + }, + "advance": { + "series": "results/zed-nix/cadence-rolling-advance", + "run_id": 37323174641, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "cache_scope": "cadence-20261005-zed-nix", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + } + ] +} diff --git a/migration/cadence-schedule-inventory-2026-10-05.json b/migration/cadence-schedule-inventory-2026-10-05.json new file mode 100644 index 000000000..841964a24 --- /dev/null +++ b/migration/cadence-schedule-inventory-2026-10-05.json @@ -0,0 +1,765 @@ +{ + "schema_version": 1, + "observed_on": "2026-10-05", + "state": "observed; historical schedules unchanged", + "repositories": [ + { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "head_sha": "256f12b07ef648695134915651c1e03cf58cfd27", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "0e6e09dec41131d244295366fee4439f7b5673ff", + "name": "CLI Canary", + "schedule": [ + { + "cron": "2 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/chroma-fresh-benchmark.yml", + "blob_sha": "d6cfb307098fa27d68a35b61ab4eeb47a7147388", + "name": "Chroma Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "94bb5ec56874b55fb3d8cc68c134a58e0aff5636", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "1,31 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-deno", + "branch": "main", + "head_sha": "bc8769a7f3eac04392354e710f39b322323a82f0", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "eacd2e8581720d6c3d94f40d259d9bb1676dac43", + "name": "CLI Canary", + "schedule": [ + { + "cron": "3 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "e7b2672f5154fce0ee7a8d219a7680c1d2cf21c7", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "2,32 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "head_sha": "12a5323f222b7faf2c30bad5c21f4441dc233fb4", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "b045a246fd6123162d847d1fd2a2618991bcedf6", + "name": "CLI Canary", + "schedule": [ + { + "cron": "5 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/duckgres-fresh-benchmark.yml", + "blob_sha": "a746c482d1e9ea6cdd5a4e62122fa97e886efd8d", + "name": "Duckgres Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "f59b3a6fd1d3f5e70c202cdacedfe4c3ddd97a30", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "5,35 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "head_sha": "b135b70ad1349cd7993f53fe70dd01353885772d", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "72561e3a4d68432dcc9ee05ab311742580c60c9f", + "name": "CLI Canary", + "schedule": [ + { + "cron": "7 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/grpc-bazel-fresh-benchmark.yml", + "blob_sha": "e6a91c9fa2ed76a9cefbe89090d9fc88d30e57a3", + "name": "gRPC Bazel Fresh", + "schedule": [ + { + "cron": "0 3 * * 5" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "06633f53ceb3b98a58cc5a5f4dff859fd1d9598b", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "7,37 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "head_sha": "ecc41a3b3e250f5268000ddde142d61a15959e95", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "3437d0f6324e54b47df5d438bd6cfa6e96731ba8", + "name": "CLI Canary", + "schedule": [ + { + "cron": "9 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/hugo-fresh-benchmark.yml", + "blob_sha": "669a6acb6dd9aedb32a690c431bc60878be63c49", + "name": "Hugo Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "b18761d26ae89ee4ba65aabf1519104f0d0baf59", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "9,39 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "head_sha": "d6186956df7f400627bb2566bd907431801b191c", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "fc672d87343291eba494deaf13219a55df60e282", + "name": "CLI Canary", + "schedule": [ + { + "cron": "11 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/hugo-go-fresh-benchmark.yml", + "blob_sha": "a0b5eded84c1dea44b276c265ccb66d6f88ab50d", + "name": "Hugo Go Fresh", + "schedule": [ + { + "cron": "0 4 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "a9afa0c852ccb9ea423715f5448c6327db6ed2dc", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "10,40 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "head_sha": "46108490e1df3ac6edbe7e75fe220b25aab155eb", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "e1fbebad9cd370fdb2abcb341fd64debd4d5b979", + "name": "CLI Canary", + "schedule": [ + { + "cron": "12 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/immich-fresh-benchmark.yml", + "blob_sha": "b79f86d331337004a1288fdb6f88cb9387ab26d7", + "name": "Immich Fresh", + "schedule": [ + { + "cron": "0 3 * * 2" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "91bc18a30a993893f2cff0814995dee6dbce34c9", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "12,42 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "head_sha": "574b7c5b2a7f138fa2cba78dbfb1a33f3448f89b", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "4843be080f498736d6b00f4f63d602aa9df16053", + "name": "CLI Canary", + "schedule": [ + { + "cron": "14 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/linkerd2-fresh-benchmark.yml", + "blob_sha": "9520528d53ec96f56024f8fb54783e16d348eab8", + "name": "Linkerd2 Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "f6ccdc84f2b66c67163afd9d6f3cbc5069018852", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "13,43 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "head_sha": "2a9f1c12c8099b3c4094028bcdc9801237924fd0", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "26016bac6319ff72f486e3ef56f3493a486a73ec", + "name": "CLI Canary", + "schedule": [ + { + "cron": "16 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/mastodon-fresh-benchmark.yml", + "blob_sha": "1add987ea7b0c126182287a7476c805547e58d4c", + "name": "Mastodon Docker Fresh", + "schedule": [ + { + "cron": "0 4 * * 2" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "3fcf7fcac070ae5b1d87ce845f940eb37b8b5b99", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "15,45 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "head_sha": "52a6cb8d9acc3d569a8c156478242b85ffd2ff19", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "864b20860a0caf8c3038c392e38458dc536f2b8f", + "name": "CLI Canary", + "schedule": [ + { + "cron": "18 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/n8n-docker-fresh-benchmark.yml", + "blob_sha": "7fd5a970403e29deeae2acac3bb12587c32c1e57", + "name": "n8n Docker Fresh", + "schedule": [ + { + "cron": "30 3 * * 6" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/n8n-fresh-benchmark.yml", + "blob_sha": "774595c682017712b55deb7f20e02f60d232687a", + "name": "n8n Turbo Fresh", + "schedule": [ + { + "cron": "0 3 * * 6" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "d53eb06298a1f306694c00820e71680a3c26f34b", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "17,47 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "head_sha": "42d22b7f50da7e225effcb9ec9a2aa97c47f97cb", + "scheduled_workflows": [ + { + "path": ".github/workflows/sync.yml", + "blob_sha": "c81c212fa2d9b2e8cc5591ae41e5bc9cd7dfdaa9", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "18,48 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-opentelemetry-java", + "branch": "main", + "head_sha": "b83cdf213f54cc9572f122d0dfbf40087e1d15fd", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "496246c8fc77d89c7e1e419c1ff24afb1e57db32", + "name": "CLI Canary", + "schedule": [ + { + "cron": "20 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/opentelemetry-java-gradle-fresh-benchmark.yml", + "blob_sha": "d90e47f33890233759446efb76e3139c068a4c62", + "name": "OpenTelemetry Java Gradle Fresh", + "schedule": [ + { + "cron": "0 3 * * 4" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "fed2ce18754929d40cc75c0fcb731d56e05e97ae", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "20,50 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "head_sha": "0462819802f2b8a6393ee8221e2bc595f8b8f058", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "b5a974165b5eb4ee8b2d721c3da61096f43654c0", + "name": "CLI Canary", + "schedule": [ + { + "cron": "21 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/posthog-fresh-benchmark.yml", + "blob_sha": "64da0626ed0435e003d25afc524eeafee574d67c", + "name": "PostHog Fresh", + "schedule": [ + { + "cron": "0 3 * * 3" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "862f73e4e5bfe01ba86dc16886fe46be6eb3074f", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "21,51 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-qdrant", + "branch": "main", + "head_sha": "6dc192c834f1ac641969dbc49a0f352369e05a18", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "886ad9d34bd80bb81c606501ec8dc422e9a5d0d8", + "name": "CLI Canary", + "schedule": [ + { + "cron": "23 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/qdrant-fresh-benchmark.yml", + "blob_sha": "60e5e575d08b285886f782764b808c4059042d3c", + "name": "Qdrant Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "c77b7926b59082e33f988d30a42ff49ec3efc866", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "23,53 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-spring-ai", + "branch": "main", + "head_sha": "a7c6468d2f7c4d7d6f96692e52875c7b4c6b499b", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "3746cb6f27d3de5c90cc684005edde78c14d6b26", + "name": "CLI Canary", + "schedule": [ + { + "cron": "25 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/spring-ai-maven-fresh-benchmark.yml", + "blob_sha": "de595ad12aa86d8dd7dce8764fca9f9779f14ff2", + "name": "Spring AI Maven Fresh", + "schedule": [ + { + "cron": "0 4 * * 4" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "34054e8aa5851c8b5eb594d5e469b5de25d5de2e", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "25,55 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "head_sha": "681058e746f602b8d0717536fa5ccfee6a2330f7", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "736511aa88aa5dd42c64215491c7e02b646705f8", + "name": "CLI Canary", + "schedule": [ + { + "cron": "27 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/storybook-fresh-benchmark.yml", + "blob_sha": "b3c7bc31db4c67ede4023204720a5474411751c6", + "name": "Storybook Nx Fresh", + "schedule": [ + { + "cron": "0 4 * * 3" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "52555528e53ffcbc6542f2f48014455837d3aac4", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "26,56 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-zed", + "branch": "main", + "head_sha": "a492769ef83da59ab47aa6a703910d64fb4d9eba", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "3a131e6f58d054993ab078a58a9a57c549169910", + "name": "CLI Canary", + "schedule": [ + { + "cron": "29 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "2c888b5a5cb32e6cdecf727bdb38be30bbc91851", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "28,58 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + } + ] +} diff --git a/migration/evidence.json b/migration/evidence.json index 58bba729d..0fab25bf3 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -16,6 +16,20 @@ "partial": 1 }, "unavailable_reference_count": 1 + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37296214039, + "verified_files": 15, + "gaps": [], + "sha256": "02cedef26b2bf4a6e4fd58b6ba454be3d6ea10e8a2daf6c08d92e511abdb5a67", + "bundle_bytes": 6100745, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37296214039.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This failed run, including available attempts, jobs, logs, artifacts, commit and workflow" } ], "exports": [ @@ -1653,6 +1667,938 @@ "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", "verified_on": "2026-10-05", "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37315116541, + "verified_files": 9, + "gaps": [], + "sha256": "ca9909f44c6315270a806f95c2d2707e64e3add3d79fb6415722097b0b3d0ba5", + "bundle_bytes": 31037, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37315116541.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37320867908, + "verified_files": 22, + "gaps": [], + "sha256": "f7a9664a0b17b2d2d448c99710d15514f1e30545bcfaa43fa9b34956379d2b3f", + "bundle_bytes": 1101686405, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320867908.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321566845, + "verified_files": 48, + "gaps": [], + "sha256": "a692391d0263c7dd9de5f4da74afbc2b7e71f9ad73ee9b0fa3ec1c687800b974", + "bundle_bytes": 1097296, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321566845.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321739971, + "verified_files": 12, + "gaps": [], + "sha256": "8d242f66c0eb47d5a45b0ae82c6a87f072820fb2b21a85053633a0000d805601", + "bundle_bytes": 275449, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321739971.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321744434, + "verified_files": 12, + "gaps": [], + "sha256": "cfad4bc4a1ea4794335742bf5f7b72068df1be53f5829588e0be7e0be5ef703b", + "bundle_bytes": 827657, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321744434.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321748690, + "verified_files": 12, + "gaps": [], + "sha256": "fec6d35de4f43c5107db22a06d8709872f9b0974845b2c61622ba08f4540d9d6", + "bundle_bytes": 156444, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321748690.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321753318, + "verified_files": 12, + "gaps": [], + "sha256": "943f1aed20b84a20ae7bd3e66c867e9aa377770748c091f494f890265b4acf15", + "bundle_bytes": 1539621, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321753318.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321758035, + "verified_files": 12, + "gaps": [], + "sha256": "604aca8af1d1211ccdf93bb62aa955eca12b949000b647309e76f75889890b73", + "bundle_bytes": 1595003, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321758035.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321762921, + "verified_files": 12, + "gaps": [], + "sha256": "099866cf3894bf3f24d99237c56d567e9a891d69795a3618b6db55f81fe248e0", + "bundle_bytes": 1469723, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321762921.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321767409, + "verified_files": 12, + "gaps": [], + "sha256": "333d355c184ced314904c188faacd8c8f128ca3009af4b07eedfdbfbdf5e3f7c", + "bundle_bytes": 329937, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321767409.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321772493, + "verified_files": 12, + "gaps": [], + "sha256": "8ea634086f30e9d31421ee5001759f28d6a5abb1835a8663c786ccc2bf420cb0", + "bundle_bytes": 347721, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321772493.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322592957, + "verified_files": 14, + "gaps": [], + "sha256": "7f67f53fa6cbfdf9e84fb54b78bdf084f38d79294e2279f671cba2df13e6053c", + "bundle_bytes": 191113, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322592957.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322598011, + "verified_files": 14, + "gaps": [], + "sha256": "d2bb19c69a726dafafb9cc998a8c54c0a442e6c2225bffc6d5d95eb0d4883acf", + "bundle_bytes": 220421, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322598011.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322604229, + "verified_files": 17, + "gaps": [], + "sha256": "592819345698de92cb117fdfa12e5e503830e3d477686667057c535450b9bf9f", + "bundle_bytes": 1100939913, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322604229.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322609279, + "verified_files": 14, + "gaps": [], + "sha256": "b150fa6c1b18923d658c39b2682f9bec768c99c58b3e47c3436e7267e4924507", + "bundle_bytes": 326834, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322609279.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322616359, + "verified_files": 11, + "gaps": [], + "sha256": "763f7a67eeb11a7709d74e2428ff656a0ede56664a81e7088dbe61cf6a42a798", + "bundle_bytes": 338991, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322616359.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322621030, + "verified_files": 12, + "gaps": [], + "sha256": "bffc77b9bc836074a77383fe5ef6ff5974def9d16ce6600c6f3a17c3041a1572", + "bundle_bytes": 366811, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322621030.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322625973, + "verified_files": 14, + "gaps": [], + "sha256": "82ec59c02b6341663b1dace65e372ff15dea6c9bb727b179402a2e4e643eb7ec", + "bundle_bytes": 115482, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322625973.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322630658, + "verified_files": 14, + "gaps": [], + "sha256": "bd5238de08f55c051c10af54129672dd68a321eb5767b76bf378a7c9c35076d5", + "bundle_bytes": 113197, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322630658.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322979117, + "verified_files": 12, + "gaps": [], + "sha256": "67c1ce2b89ccb76d94bb7e0b3bce59bde293ca308c17cc33b991f75f6a7eb933", + "bundle_bytes": 826739, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322979117.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322983836, + "verified_files": 12, + "gaps": [], + "sha256": "cb6fe435b0385ecbd9bf908a995caca4057219763a1a40cf32a7235aa731911d", + "bundle_bytes": 154007, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322983836.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323107836, + "verified_files": 9, + "gaps": [], + "sha256": "c03c24b601c1a8e5bb5665974244c8a1909a8c4baa39fef7af98e24135fbf668", + "bundle_bytes": 84529, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323107836.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323112601, + "verified_files": 11, + "gaps": [], + "sha256": "cd0ec90eec324ba58a7d0057e07f88479d748b06f2e2767e95a24c7e84e36a9c", + "bundle_bytes": 145242, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323112601.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323117664, + "verified_files": 17, + "gaps": [], + "sha256": "a6c81e98459285213015dcab97e58811e265b2ea14b05a14033bed8096822780", + "bundle_bytes": 1101082613, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323117664.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323121773, + "verified_files": 12, + "gaps": [], + "sha256": "5801a2d2a574d29f3b9bf57a498b8affc4149fa9e13b799cd654c4da24cde71c", + "bundle_bytes": 282104, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323121773.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323126507, + "verified_files": 12, + "gaps": [], + "sha256": "1ad5f7e4ca2684465c84f7dd8718aae2278ff16fbdcefd190771cb6eb000dbe9", + "bundle_bytes": 435959, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323126507.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323131290, + "verified_files": 12, + "gaps": [], + "sha256": "61e572577342afc628eeefd6991964b60d30f6e309365f8567447366ff6e6bea", + "bundle_bytes": 146637, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323131290.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323136088, + "verified_files": 11, + "gaps": [], + "sha256": "72b85d786bdf715a8a1e2c9ca796a8100a1115cdc6d33ad50c5172a7a5ed429c", + "bundle_bytes": 173633, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323136088.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323140444, + "verified_files": 12, + "gaps": [], + "sha256": "df570b5c90c61cc8361a3417b07993243388a5b14027680c0a95b842488293ee", + "bundle_bytes": 1521595, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323140444.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323145174, + "verified_files": 12, + "gaps": [], + "sha256": "a9eb1b54d32882655857258e64fd3502748dba73d6dbc965d4e002db0fc9320f", + "bundle_bytes": 1492015, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323145174.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323150431, + "verified_files": 12, + "gaps": [], + "sha256": "7b9e55505c6e141641973baabe03b5d10601bcd96902f4a198492212ef8cd750", + "bundle_bytes": 1494733, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323150431.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323155106, + "verified_files": 12, + "gaps": [], + "sha256": "92d86e6093a84762c6f520e492db177f37e367f8707a284d528a211791c2efc6", + "bundle_bytes": 342132, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323155106.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323159608, + "verified_files": 11, + "gaps": [], + "sha256": "529277911178291c805dcd36b9e2b05c5957f94125071f76db212acd73dfd136", + "bundle_bytes": 100977, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323159608.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323169828, + "verified_files": 11, + "gaps": [], + "sha256": "c401fac1f49c571d64e22854bebbcbbacd68f625c7b6054994da260487c22461", + "bundle_bytes": 97016, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323169828.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323234591, + "verified_files": 12, + "gaps": [], + "sha256": "004555369daa75edae6aa4f987b1a05f54145d02e2e6bb8acd0604aca64643d2", + "bundle_bytes": 359874, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323234591.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323465858, + "verified_files": 14, + "gaps": [], + "sha256": "53ff9d72bdff5910a049abd83288b595b4776db3296f0913f432fb46a6240b6b", + "bundle_bytes": 556245, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323465858.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324016178, + "verified_files": 14, + "gaps": [], + "sha256": "7c2234a43786944d100dd1a4af90e8394e94484ecd6936727223fc04f070402e", + "bundle_bytes": 212147, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324016178.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324020999, + "verified_files": 14, + "gaps": [], + "sha256": "6eb956103b370f83cb0e254de9536acb03938442ff1b62496e1e96ed37ef2676", + "bundle_bytes": 107468, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324020999.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324025714, + "verified_files": 14, + "gaps": [], + "sha256": "c0d5b3968fd3baef55c41533cc727b127ff5889094cd510e50f8db4f6f43ae18", + "bundle_bytes": 104968, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324025714.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324032100, + "verified_files": 14, + "gaps": [], + "sha256": "cab8cfbc72be15f995b06274a9b01d24f0fb219773c288700c10b7f266cb387a", + "bundle_bytes": 183124, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324032100.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324036688, + "verified_files": 14, + "gaps": [], + "sha256": "1f47fb24f0bebbb20f5c74433ba379aa9908839ee9e2af384404ca1349d5e8fc", + "bundle_bytes": 318399, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324036688.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324204556, + "verified_files": 14, + "gaps": [], + "sha256": "4fc9961f582cef0bce623ead2d3931b4bdc98e5169d35dff8bbae939285f84b8", + "bundle_bytes": 161465, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324204556.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324209639, + "verified_files": 10, + "gaps": [], + "sha256": "9221d4046fb51bf1ebf63b18851ff4ea7e51cb968ea41cedb8dfea7b8ee38158", + "bundle_bytes": 153230, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324209639.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324214890, + "verified_files": 14, + "gaps": [], + "sha256": "ebeda092c2c2ff2c299f2ea0e4aabb119796f5f3e718bf40c6bc1fe3291e496f", + "bundle_bytes": 258679, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324214890.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324219008, + "verified_files": 14, + "gaps": [], + "sha256": "d4fed72d12b3a4192f88aa482ae3704467909296d734d4ebed6dc898112e4d89", + "bundle_bytes": 550952, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324219008.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324223748, + "verified_files": 10, + "gaps": [], + "sha256": "cad3872631c0ee88d449536487e595e30742ee6d5d0a07b3fca8a885ea58bca3", + "bundle_bytes": 95227, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324223748.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324228460, + "verified_files": 14, + "gaps": [], + "sha256": "8c51c61558e0ff24b43c70ee64991636577e90408ce3db6e7472f91d603b68e6", + "bundle_bytes": 106198, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324228460.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325135541, + "verified_files": 14, + "gaps": [], + "sha256": "62fb0f146fa5feb568034f503c814b5aba1016805872c1a16500a668be352c94", + "bundle_bytes": 209949, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325135541.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325140807, + "verified_files": 14, + "gaps": [], + "sha256": "cc1af7a88f60ba75173a25bb0ffad5e6081816b66234b3529c589f2acff7e0be", + "bundle_bytes": 103962, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325140807.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325183094, + "verified_files": 12, + "gaps": [], + "sha256": "c38518b43c2552fca4c2300c251f48f84da443d1fd216c24dfdf012f65a9123a", + "bundle_bytes": 327800, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325183094.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325282767, + "verified_files": 14, + "gaps": [], + "sha256": "35f5c6ae08f20dddebe4d2520b0341f30f1f79106de6e8fb520eb1244c7862bf", + "bundle_bytes": 211243, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325282767.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325287254, + "verified_files": 14, + "gaps": [], + "sha256": "2223af0b476341716133ce0221e569b55c073ead9304f21ea5eda60c70d93b33", + "bundle_bytes": 105379, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325287254.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37326133177, + "verified_files": 14, + "gaps": [], + "sha256": "11526bdc0cf3d01e989928b3ee6aef7d7636bd2ad495ff1b1f769abde866019d", + "bundle_bytes": 363454, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37326133177.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37326340221, + "verified_files": 14, + "gaps": [], + "sha256": "0eb74a37aaa8137d5592c88eadd00cfc406afb40a7cf6128991dbe15a0115c20", + "bundle_bytes": 361816, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37326340221.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323164071, + "verified_files": 12, + "gaps": [], + "sha256": "ff8ac7d32632dd15f7cd666a65b959b6c4c5b542bb2160485fac9f8cf6494553", + "bundle_bytes": 283616, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323164071.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325075463, + "verified_files": 22, + "gaps": [], + "sha256": "d762c7f83e8e840169fbd71436ba075cd617ebf465d7fea7acf0f39a754a8163", + "bundle_bytes": 1101648302, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325075463.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37327069343, + "verified_files": 22, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "31428770e26114ffca4544d0a1690b3788d06347bbda9deb75e2ceb4969ee04d", + "archive_bytes": 4449826654, + "archive_sha256": "6d8b1992002a2a8eddd65bf7b907a86d6a253ace6d6781d033164cc11c9eec29", + "bundle_parts": [ + { + "name": "benchmarks-37327069343.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.part001", + "bytes": 1900000000, + "sha256": "b602d6ca0db9d499dbcd01cdd468c7e2daccd0092b75edab5d3965e02d35ad4b" + }, + { + "name": "benchmarks-37327069343.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.part002", + "bytes": 1900000000, + "sha256": "95af559c1621c2d4da752dff1ee071eaf29fe78c036e154d37d7fcc7f62ea8ea" + }, + { + "name": "benchmarks-37327069343.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.part003", + "bytes": 649826654, + "sha256": "840dc324c77d027725f1206ddc21b088f8eca1dcf5d5a765183f1e692d150640" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37320869828, + "verified_files": 22, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "439a6c1df950844804eff84b809058f83c942adaffab05f57898fcfe2f2ca21a", + "archive_bytes": 4449816162, + "archive_sha256": "eb77ba7aa040779bf26e7c392908510da97f2c69661012f5e35e3135bb12e631", + "bundle_parts": [ + { + "name": "benchmarks-37320869828.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.part001", + "bytes": 1900000000, + "sha256": "2c2dcf9728a1046176a60b3eff3a9dcf7f8dd96a741f43e35946c28713f54639" + }, + { + "name": "benchmarks-37320869828.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.part002", + "bytes": 1900000000, + "sha256": "116e6a0a753bd348572de3b7415212dd2ba8e10d26a7bc7d3c2f11d39e3ca2fc" + }, + { + "name": "benchmarks-37320869828.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.part003", + "bytes": 649816162, + "sha256": "b4d202109538cdadc5854454497be2a0c25daa137c790a3509f51612723c742c" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323174641, + "verified_files": 17, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "97f8de554fecc282a1aec272ee0144f9c1b37340b363d2608729f73c4105e83b", + "archive_bytes": 4399876552, + "archive_sha256": "48b7c9ecea8aeae7137e5993b1b69a2d68bcadc85c9e671995ca11c760991064", + "bundle_parts": [ + { + "name": "benchmarks-37323174641.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.part001", + "bytes": 1900000000, + "sha256": "0c81dcfc13451dad46a56166d05949db7c10fc6a056ad63c30e4ca041e90cdc7" + }, + { + "name": "benchmarks-37323174641.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.part002", + "bytes": 1900000000, + "sha256": "fba47b80d34745ec7dc91c2bd6ca2fa40915b2d75ca8ac72bf81e94aea99e023" + }, + { + "name": "benchmarks-37323174641.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.part003", + "bytes": 599876552, + "sha256": "c1e3a6d1ba77c928ac09a55fa4c59b0fcc879220344bb93a56189ce6601cfaa0" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322636399, + "verified_files": 17, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "1567a96ab8c6dacd98c285aadf34baae4b9ebe388ffcc35bdeb73a8d3e21e6da", + "archive_bytes": 4449189479, + "archive_sha256": "080b578d8fd21f65c302a24cce44f03770df87c8b3a7ecfb14e3f9f745819da9", + "bundle_parts": [ + { + "name": "benchmarks-37322636399.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.part001", + "bytes": 1900000000, + "sha256": "883a9acbdc5b5b3187e16dc719471393ef6b2762d539d2ee9213041344ecb53d" + }, + { + "name": "benchmarks-37322636399.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.part002", + "bytes": 1900000000, + "sha256": "4d711d9c693e459f98e9391ec923c1e53d3fff88c6b207dbf78506ba1f2a9e39" + }, + { + "name": "benchmarks-37322636399.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.part003", + "bytes": 649189479, + "sha256": "a3d32aa094b484d950e84530dadc17e29185c81bf11d8f8a29bc6a4f0ffcbc09" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" } ] } diff --git a/migration/rehearsals/mastodon-publication/commits.json b/migration/rehearsals/mastodon-publication/commits.json new file mode 100644 index 000000000..94fe3286a --- /dev/null +++ b/migration/rehearsals/mastodon-publication/commits.json @@ -0,0 +1 @@ +[{"message":"Record mastodon rolling completed","sha":"540010c2ef47f38ecd2556b53830a27f053dc7fd","verified":true},{"message":"Record mastodon rolling requested","sha":"44b9954102b65bfc370bd78c6e8123ae999fa2a9","verified":true},{"message":"Record mastodon rolling dispatching","sha":"914e37913a240929d68908bb0e5b46db3c898ad9","verified":true}] diff --git a/migration/rehearsals/mastodon-publication/concurrency-after-fix.json b/migration/rehearsals/mastodon-publication/concurrency-after-fix.json new file mode 100644 index 000000000..36b8129c0 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/concurrency-after-fix.json @@ -0,0 +1,26 @@ +[ + { + "writer": "a", + "commit": "dc13d9d6712e088c7274d3bbd530e617c7989ebb", + "conflicts": [] + }, + { + "writer": "b", + "commit": "d06fa3645dc73f0ced3f296d38b757e91cd424a4", + "conflicts": [ + { + "type": "FORBIDDEN", + "path": [ + "createCommitOnBranch" + ], + "locations": [ + { + "line": 1, + "column": 48 + } + ], + "message": "is at dc13d9d6712e088c7274d3bbd530e617c7989ebb but expected 332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff" + } + ] + } +] diff --git a/migration/rehearsals/mastodon-publication/concurrency-before-fix.json b/migration/rehearsals/mastodon-publication/concurrency-before-fix.json new file mode 100644 index 000000000..18b6a3d21 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/concurrency-before-fix.json @@ -0,0 +1,26 @@ +[ + { + "writer": "a", + "error": "Source publication failed: is at 332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff but expected dbb5e9401a676589cfeabef936bcf08436bb0e10", + "conflicts": [ + { + "type": "FORBIDDEN", + "path": [ + "createCommitOnBranch" + ], + "locations": [ + { + "line": 1, + "column": 48 + } + ], + "message": "is at 332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff but expected dbb5e9401a676589cfeabef936bcf08436bb0e10" + } + ] + }, + { + "writer": "b", + "commit": "332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff", + "conflicts": [] + } +] diff --git a/migration/rehearsals/mastodon-publication/plan.md b/migration/rehearsals/mastodon-publication/plan.md new file mode 100644 index 000000000..92ac79b54 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/plan.md @@ -0,0 +1,18 @@ +# Source publication rehearsal + +Question: does the source publisher conditionally advance an isolated branch, +retain intent before dispatch, request both declared Mastodon rolling variants +with the exact canary, and reconcile their returned run IDs without modifying +main? + +The input is the Mastodon proposal verified by source-check run 37321566845. +Use branch cadence-rehearsal-publication and CLI vcli-canary-7a5b27146ebe. +Request server and streaming once each, with their declared GitHub Actions and +BoringCache providers. Use the branch's default rolling cache cohort. These are +bootstrap observations, not evidence of changed-source cache reuse. The separate +cadence-rolling seed and advancement series test cache reuse. + +Verify the published file diff, commit verification, exact returned run IDs, +retained intent and final receipts, reconciliation outcome, and unchanged main +head and activation setting. Retain failures and uncertain requests. Do not retry +an uncertain dispatch automatically. diff --git a/migration/rehearsals/mastodon-publication/proposal.json b/migration/rehearsals/mastodon-publication/proposal.json new file mode 100644 index 000000000..6013082b3 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/proposal.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "records": [ + { + "case_id": "mastodon", + "updated": true, + "base_sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "head_sha": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "state": "proposal", + "requires_case_qualification": true + } + ], + "publication": "requires-review" +} diff --git a/migration/rehearsals/mastodon-publication/receipt.json b/migration/rehearsals/mastodon-publication/receipt.json new file mode 100644 index 000000000..54c9264e0 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/receipt.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "state": "completed", + "cli_version": "vcli-canary-7a5b27146ebe", + "ref": "cadence-rehearsal-publication", + "proposal": { + "case_id": "mastodon", + "updated": true, + "base_sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "head_sha": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "state": "proposal", + "requires_case_qualification": true + }, + "created_at": "2026-10-05T14:13:24Z", + "runs": [ + { + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "variant": "server" + }, + "state": "requested", + "id": 37322979117, + "url": "https://github.com/boringcache/benchmarks/actions/runs/37322979117", + "outcome": "success" + }, + { + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "variant": "streaming" + }, + "state": "requested", + "id": 37322983836, + "url": "https://github.com/boringcache/benchmarks/actions/runs/37322983836", + "outcome": "success" + } + ] +} diff --git a/migration/rehearsals/mastodon-publication/review.md b/migration/rehearsals/mastodon-publication/review.md new file mode 100644 index 000000000..032ee3c30 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/review.md @@ -0,0 +1,29 @@ +# Source publication rehearsal result + +The publisher advanced only `cadence-rehearsal-publication`. It committed intent +before making requests, retained both returned run IDs, then reconciled both +successful outcomes to `completed`. All three publication commits are verified +by GitHub. The changed paths were Mastodon's case definition and its current and +per-source rolling receipts. + +- Server: [run 37322979117](https://github.com/boringcache/benchmarks/actions/runs/37322979117). +- Streaming: [run 37322983836](https://github.com/boringcache/benchmarks/actions/runs/37322983836). +- Final receipt: [commit 540010c2](https://github.com/boringcache/benchmarks/commit/540010c2). + +The main branch remained at `b3855db2e9c0b27636cc143a772fe9917caf4940` throughout +this rehearsal, and `BENCHMARK_CADENCE_ACTIVE` remained unset. These are bootstrap +runs and do not establish changed-source cache reuse or comparative performance. + +The local rehearsal used the operator's authenticated GitHub CLI. The separate +hosted source rehearsal exercised the read-only inspector and dry-run publisher +with the workflow token; it did not exercise main-branch write permissions. +The first activated source cycle must verify those writes and its retained +receipts. No production activation is implied by this rehearsal. + +A concurrent-write check forced two publishers to read the same branch head. +The first check exposed a ref-lock race reported by GitHub as `FORBIDDEN` with +exact current and expected commit SHAs. The publisher now retries that specific +conflict as well as `STALE_DATA`, while rejecting ordinary permission failures. +The repeated check committed both independent files; one publisher encountered +and recovered from that exact conflict. The before/after responses remain beside +this review. These checks modified only the isolated rehearsal branch. diff --git a/results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json b/results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json new file mode 100644 index 000000000..43c3d5027 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324204556", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "cabad803e46d684f98d37e05d6725817a86d6d9e279f8b49a7185bb3e99312d6" +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..d855b69ef --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324204556, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556" +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/report.json b/results/executorch-buck2/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..d3c3b8096 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "f73f472febc211a9b27ff0d819c5a6a25c6db9a28aa614f194ef7c1e5f043650", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324204556", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "cabad803e46d684f98d37e05d6725817a86d6d9e279f8b49a7185bb3e99312d6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10076324300001716, + "build_seconds": 27.035207954000043, + "workflow_seconds": null, + "total_seconds": 27.13597119700006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003453705999959311, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.13597119700006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174578", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] + }, + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20143210499998077, + "build_seconds": 21.858722754999974, + "workflow_seconds": null, + "total_seconds": 22.060154859999955, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9866127600000141, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 22.060154859999955 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174579", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/report.md b/results/executorch-buck2/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..762d40cae --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/report.md @@ -0,0 +1,30 @@ +# executorch-buck2: cadence-rolling-advance-02 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10076324300001716 | 27.035207954000043 | 27.13597119700006 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20143210499998077 | 21.858722754999974 | 22.060154859999955 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..50f7f78f3 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10076324300001716, + "build_seconds": 27.035207954000043, + "workflow_seconds": null, + "total_seconds": 27.13597119700006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003453705999959311, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.13597119700006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174578", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f97fcd1d4 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20143210499998077, + "build_seconds": 21.858722754999974, + "workflow_seconds": null, + "total_seconds": 22.060154859999955, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9866127600000141, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 22.060154859999955 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174579", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/series.json b/results/executorch-buck2/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..27531ffea --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "f73f472febc211a9b27ff0d819c5a6a25c6db9a28aa614f194ef7c1e5f043650" +} diff --git a/results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json b/results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json new file mode 100644 index 000000000..c4d5e8438 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323107836", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled", + "bazel-remote executorch-buck2 commit: cancelled", + "BoringCache executorch-buck2 commit: cancelled", + "${{ inputs.case_id }} rolling report: cancelled" + ], + "evidence_manifest_sha256": "8560277d3446809f06cdfa9491df840954477e4305218e6acee783a31e19d1ea" +} diff --git a/results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..2564b5b40 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:28Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323107836, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836" +} diff --git a/results/executorch-buck2/cadence-rolling-advance/report.json b/results/executorch-buck2/cadence-rolling-advance/report.json new file mode 100644 index 000000000..7f6149c0b --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/report.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "plan_sha256": "b5a60430a6aea0bbc4b78b616f032125887f7f10628313235f9d316c4919900f", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323107836", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled", + "bazel-remote executorch-buck2 commit: cancelled", + "BoringCache executorch-buck2 commit: cancelled", + "${{ inputs.case_id }} rolling report: cancelled" + ], + "evidence_manifest_sha256": "8560277d3446809f06cdfa9491df840954477e4305218e6acee783a31e19d1ea" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323107836", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled", + "bazel-remote executorch-buck2 commit: cancelled", + "BoringCache executorch-buck2 commit: cancelled", + "${{ inputs.case_id }} rolling report: cancelled" + ], + "evidence_manifest_sha256": "8560277d3446809f06cdfa9491df840954477e4305218e6acee783a31e19d1ea" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "boringcache" + ], + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-advance/report.md b/results/executorch-buck2/cadence-rolling-advance/report.md new file mode 100644 index 000000000..62dd7a5e2 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# executorch-buck2: cadence-rolling-advance + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 0/2 recorded; 0 failed; 2 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323107836 failed completion checks: + +- Workflow concluded cancelled +- bazel-remote executorch-buck2 commit: cancelled +- BoringCache executorch-buck2 commit: cancelled +- ${{ inputs.case_id }} rolling report: cancelled + +Missing observations: + +- Sample 1, BoringCache, Changed-source build +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-advance/series.json b/results/executorch-buck2/cadence-rolling-advance/series.json new file mode 100644 index 000000000..6f2ebf22e --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e022775af4acc3da2ff2916716d1ec5f3f7d6021bc07325f91e703b93af37647", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b5a60430a6aea0bbc4b78b616f032125887f7f10628313235f9d316c4919900f" +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json b/results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json new file mode 100644 index 000000000..1662ae814 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324032100", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3e4e6174c3c5bff4a09c840f4d8d17aa6dfc60f48a6100b307938ea86811f13c" +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..5eed2dfe7 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:22Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324032100, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100" +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/report.json b/results/executorch-buck2/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..77610a6c9 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "faeaeb496d8118d98a9bad0fe2fb23a693f95f964e36df35d46b3e0175e0a981", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324032100", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3e4e6174c3c5bff4a09c840f4d8d17aa6dfc60f48a6100b307938ea86811f13c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010354490000509, + "build_seconds": 158.649545483, + "workflow_seconds": null, + "total_seconds": 158.75058093200005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004191613999978472, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 158.75058093200005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174533", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] + }, + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20147264299998824, + "build_seconds": 161.122134534, + "workflow_seconds": null, + "total_seconds": 161.32360717699999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 2.3798232430000326, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 161.32360717699999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174532", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/report.md b/results/executorch-buck2/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..5375f7d9f --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# executorch-buck2: cadence-rolling-seed-02 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1010354490000509 | 158.649545483 | 158.75058093200005 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20147264299998824 | 161.122134534 | 161.32360717699999 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..a2ff00677 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010354490000509, + "build_seconds": 158.649545483, + "workflow_seconds": null, + "total_seconds": 158.75058093200005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004191613999978472, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 158.75058093200005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174533", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..8ba8054a1 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20147264299998824, + "build_seconds": 161.122134534, + "workflow_seconds": null, + "total_seconds": 161.32360717699999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 2.3798232430000326, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 161.32360717699999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174532", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/series.json b/results/executorch-buck2/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..eaa6d8733 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "faeaeb496d8118d98a9bad0fe2fb23a693f95f964e36df35d46b3e0175e0a981" +} diff --git a/results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json b/results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json new file mode 100644 index 000000000..cd1f43ade --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322592957", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322592957", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ef10d1633ac3cbe6e9b49652f2d4e421147ab1558ec026da88982303d2dc9521" +} diff --git a/results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..269294476 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:36Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322592957, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322592957" +} diff --git a/results/executorch-buck2/cadence-rolling-seed/report.json b/results/executorch-buck2/cadence-rolling-seed/report.json new file mode 100644 index 000000000..bc0d07159 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "35d1e5b1e7d52193a77c09c26dfc6b92ffdd4d61738a3c856ff4c5245ee3792e", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322592957", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322592957", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ef10d1633ac3cbe6e9b49652f2d4e421147ab1558ec026da88982303d2dc9521" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102946500001053, + "build_seconds": 163.48988875000003, + "workflow_seconds": null, + "total_seconds": 163.59091821500004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.005228078999948593, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 163.59091821500004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174431", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] + }, + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20142720100000133, + "build_seconds": 127.76391611900004, + "workflow_seconds": null, + "total_seconds": 127.96534332000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 3.117719557999976, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 127.96534332000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174430", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-seed/report.md b/results/executorch-buck2/cadence-rolling-seed/report.md new file mode 100644 index 000000000..0a27801da --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# executorch-buck2: cadence-rolling-seed + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10102946500001053 | 163.48988875000003 | 163.59091821500004 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20142720100000133 | 127.76391611900004 | 127.96534332000004 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6f088288f --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102946500001053, + "build_seconds": 163.48988875000003, + "workflow_seconds": null, + "total_seconds": 163.59091821500004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.005228078999948593, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 163.59091821500004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174431", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..16901156b --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20142720100000133, + "build_seconds": 127.76391611900004, + "workflow_seconds": null, + "total_seconds": 127.96534332000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 3.117719557999976, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 127.96534332000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174430", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed/series.json b/results/executorch-buck2/cadence-rolling-seed/series.json new file mode 100644 index 000000000..3989fe1dd --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "35d1e5b1e7d52193a77c09c26dfc6b92ffdd4d61738a3c856ff4c5245ee3792e" +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json b/results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json new file mode 100644 index 000000000..1379d1357 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324209639", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "c6f067228af8803ce9918774f33c07012c144d6c60330777002c855ff6af8117" +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..1856eb5e5 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324209639, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639" +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/report.json b/results/gogs-moon/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..696acf629 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/report.json @@ -0,0 +1,220 @@ +{ + "schema_version": 1, + "plan_sha256": "5660c43a1cd7acddbb845be324a33442a7786c3299af0b1d2ac0fc78a7ddec6b", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324209639", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "c6f067228af8803ce9918774f33c07012c144d6c60330777002c855ff6af8117" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324209639", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "c6f067228af8803ce9918774f33c07012c144d6c60330777002c855ff6af8117" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011285519999916, + "build_seconds": 8.481873947000011, + "workflow_seconds": null, + "total_seconds": 8.583002499000003, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001908086000000253, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.583002499000003 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324209639", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174556", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324209639-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324209639" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/report.md b/results/gogs-moon/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..3094b44ed --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/report.md @@ -0,0 +1,38 @@ +# gogs-moon: cadence-rolling-advance-02 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37324209639 failed completion checks: + +- Workflow concluded failure +- BoringCache gogs-moon commit: failure +- gogs-moon rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1011285519999916 | 8.481873947000011 | 8.583002499000003 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6564d993f --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011285519999916, + "build_seconds": 8.481873947000011, + "workflow_seconds": null, + "total_seconds": 8.583002499000003, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001908086000000253, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.583002499000003 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324209639", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174556", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324209639-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324209639" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/series.json b/results/gogs-moon/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..51cd9834d --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "5660c43a1cd7acddbb845be324a33442a7786c3299af0b1d2ac0fc78a7ddec6b" +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json b/results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json new file mode 100644 index 000000000..1c657b93b --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325282767", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2f19fc35799b4b7972eae3a033bd77e7f64127c99f6bcb801e171cfe6e8cce18" +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..ed00d000b --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3", + "series_id": "cadence-rolling-advance-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:30:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325282767, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767" +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/report.json b/results/gogs-moon/cadence-rolling-advance-03/report.json new file mode 100644 index 000000000..2fe3586f2 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "7d4cffbb92d6bdc71d107a9bd62991f93d0ee589cc6fa0914905f21e7aeb1517", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325282767", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2f19fc35799b4b7972eae3a033bd77e7f64127c99f6bcb801e171cfe6e8cce18" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10085198400000195, + "build_seconds": 7.968215224000005, + "workflow_seconds": null, + "total_seconds": 8.069067208000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001377290000007747, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.069067208000007 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174605", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20138485300000752, + "build_seconds": 8.271418910000008, + "workflow_seconds": null, + "total_seconds": 8.472803763000016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9762034230000012, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.472803763000016 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174606", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/report.md b/results/gogs-moon/cadence-rolling-advance-03/report.md new file mode 100644 index 000000000..9da06fc04 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-advance-03 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10085198400000195 | 7.968215224000005 | 8.069067208000007 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20138485300000752 | 8.271418910000008 | 8.472803763000016 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..0ef168660 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10085198400000195, + "build_seconds": 7.968215224000005, + "workflow_seconds": null, + "total_seconds": 8.069067208000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001377290000007747, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.069067208000007 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174605", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f771aa06c --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20138485300000752, + "build_seconds": 8.271418910000008, + "workflow_seconds": null, + "total_seconds": 8.472803763000016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9762034230000012, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.472803763000016 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174606", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/series.json b/results/gogs-moon/cadence-rolling-advance-03/series.json new file mode 100644 index 000000000..6d35b8e64 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-03", + "created_at": "2026-10-05T14:30:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "7d4cffbb92d6bdc71d107a9bd62991f93d0ee589cc6fa0914905f21e7aeb1517" +} diff --git a/results/gogs-moon/cadence-rolling-advance/completions/37323112601.json b/results/gogs-moon/cadence-rolling-advance/completions/37323112601.json new file mode 100644 index 000000000..83dc4a000 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/completions/37323112601.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323112601", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "2a499070ff217a6ad4e09274fac835824cb2c065a27d45aaf1efc4fc8c16483a" +} diff --git a/results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..6108a9bad --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:30Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323112601, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601" +} diff --git a/results/gogs-moon/cadence-rolling-advance/report.json b/results/gogs-moon/cadence-rolling-advance/report.json new file mode 100644 index 000000000..9578ef243 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "c7a436522dc1526a276da2991f093d32c4bf1440c6ad9110ad08ca2f7593e412", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323112601", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "2a499070ff217a6ad4e09274fac835824cb2c065a27d45aaf1efc4fc8c16483a" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323112601", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "2a499070ff217a6ad4e09274fac835824cb2c065a27d45aaf1efc4fc8c16483a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "abb49ac3a869811c5c19b69223e824cdc75f2921c309473b6ec924dda803bf89", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10094187800001464, + "build_seconds": 8.97829006400002, + "workflow_seconds": null, + "total_seconds": 9.079231942000035, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.5973653270000341, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.079231942000035 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323112601", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174469", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323112601-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323112601" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-advance/report.md b/results/gogs-moon/cadence-rolling-advance/report.md new file mode 100644 index 000000000..b4f0f3b10 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# gogs-moon: cadence-rolling-advance + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323112601 failed completion checks: + +- Workflow concluded failure +- bazel-remote gogs-moon commit: failure +- gogs-moon rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.10094187800001464 | 8.97829006400002 | 9.079231942000035 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..839227812 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "abb49ac3a869811c5c19b69223e824cdc75f2921c309473b6ec924dda803bf89", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10094187800001464, + "build_seconds": 8.97829006400002, + "workflow_seconds": null, + "total_seconds": 9.079231942000035, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.5973653270000341, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.079231942000035 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323112601", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174469", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323112601-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323112601" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance/series.json b/results/gogs-moon/cadence-rolling-advance/series.json new file mode 100644 index 000000000..adba0dc27 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "abb49ac3a869811c5c19b69223e824cdc75f2921c309473b6ec924dda803bf89", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "c7a436522dc1526a276da2991f093d32c4bf1440c6ad9110ad08ca2f7593e412" +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json b/results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json new file mode 100644 index 000000000..8fc6e7d05 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324016178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324016178", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "564fa89fa147d542879d4ad7d8b9564103bc572af9da1acd609a18d2804ccf21" +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..ac1b7e9ca --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:15Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324016178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324016178" +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/report.json b/results/gogs-moon/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..d336db5b3 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "23b7faf2d7426940a8b1dc9065cf32fc3bc7adac6f2ce1261fcf1eac2a2e9288", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324016178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324016178", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "564fa89fa147d542879d4ad7d8b9564103bc572af9da1acd609a18d2804ccf21" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111217799999395, + "build_seconds": 9.285103848000006, + "workflow_seconds": null, + "total_seconds": 9.386216026, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0053126480000003085, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.386216026 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174541", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.30168808900000954, + "build_seconds": 8.787957718000001, + "workflow_seconds": null, + "total_seconds": 9.08964580700001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.628428673000002, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.08964580700001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174540", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/report.md b/results/gogs-moon/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..ae9878ec9 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-seed-02 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10111217799999395 | 9.285103848000006 | 9.386216026 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.30168808900000954 | 8.787957718000001 | 9.08964580700001 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..772c20169 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111217799999395, + "build_seconds": 9.285103848000006, + "workflow_seconds": null, + "total_seconds": 9.386216026, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0053126480000003085, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.386216026 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174541", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..a24b99b64 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.30168808900000954, + "build_seconds": 8.787957718000001, + "workflow_seconds": null, + "total_seconds": 9.08964580700001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.628428673000002, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.08964580700001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174540", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/series.json b/results/gogs-moon/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..7caab2670 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "23b7faf2d7426940a8b1dc9065cf32fc3bc7adac6f2ce1261fcf1eac2a2e9288" +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json b/results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json new file mode 100644 index 000000000..6b08342e2 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325135541", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d926248648e2c843978677c49733b22262f18546233d52922b5e1e6b6c786bf2" +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..5d60f29fe --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3", + "series_id": "cadence-rolling-seed-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:29:33Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325135541, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541" +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/report.json b/results/gogs-moon/cadence-rolling-seed-03/report.json new file mode 100644 index 000000000..ec4f9896d --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "1778935d51f88873928e2fa318c7d98d3ce6a582be6afd3869dc2238b57d0aad", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325135541", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d926248648e2c843978677c49733b22262f18546233d52922b5e1e6b6c786bf2" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010086570000226, + "build_seconds": 8.791259834000016, + "workflow_seconds": null, + "total_seconds": 8.892268491000038, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004114332999989756, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.892268491000038 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174591", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20145711899999696, + "build_seconds": 9.786454968000044, + "workflow_seconds": null, + "total_seconds": 9.98791208700004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.7367870319999952, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.98791208700004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174590", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/report.md b/results/gogs-moon/cadence-rolling-seed-03/report.md new file mode 100644 index 000000000..281f1c931 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-seed-03 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1010086570000226 | 8.791259834000016 | 8.892268491000038 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20145711899999696 | 9.786454968000044 | 9.98791208700004 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..fae91ffa7 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010086570000226, + "build_seconds": 8.791259834000016, + "workflow_seconds": null, + "total_seconds": 8.892268491000038, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004114332999989756, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.892268491000038 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174591", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d88d0e545 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20145711899999696, + "build_seconds": 9.786454968000044, + "workflow_seconds": null, + "total_seconds": 9.98791208700004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.7367870319999952, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.98791208700004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174590", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/series.json b/results/gogs-moon/cadence-rolling-seed-03/series.json new file mode 100644 index 000000000..d9f42ab2f --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-03", + "created_at": "2026-10-05T14:29:29Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "1778935d51f88873928e2fa318c7d98d3ce6a582be6afd3869dc2238b57d0aad" +} diff --git a/results/gogs-moon/cadence-rolling-seed/completions/37322598011.json b/results/gogs-moon/cadence-rolling-seed/completions/37322598011.json new file mode 100644 index 000000000..03ab21131 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/completions/37322598011.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322598011", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322598011", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1e70ae525c9b8d7c2a0e47fb57008f595706a292bbbde0099a3862a0729a3307" +} diff --git a/results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..ac7ca3c1c --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322598011, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322598011" +} diff --git a/results/gogs-moon/cadence-rolling-seed/report.json b/results/gogs-moon/cadence-rolling-seed/report.json new file mode 100644 index 000000000..842f88978 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "1fc9fdbfd692a7c3b1c3479e4c1cc1a1f55190e7a7f4a7f781986271e4a5cee8", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322598011", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322598011", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1e70ae525c9b8d7c2a0e47fb57008f595706a292bbbde0099a3862a0729a3307" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10112419299997555, + "build_seconds": 9.287674577000018, + "workflow_seconds": null, + "total_seconds": 9.388798769999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004315487999974721, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.388798769999994 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174434", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10093500900000407, + "build_seconds": 8.487470181999996, + "workflow_seconds": null, + "total_seconds": 8.588405191, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9377041529999985, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.588405191 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174433", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-seed/report.md b/results/gogs-moon/cadence-rolling-seed/report.md new file mode 100644 index 000000000..8e1e26a25 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-seed + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10112419299997555 | 9.287674577000018 | 9.388798769999994 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.10093500900000407 | 8.487470181999996 | 8.588405191 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..8433ef3fb --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10112419299997555, + "build_seconds": 9.287674577000018, + "workflow_seconds": null, + "total_seconds": 9.388798769999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004315487999974721, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.388798769999994 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174434", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..b8d702e1d --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10093500900000407, + "build_seconds": 8.487470181999996, + "workflow_seconds": null, + "total_seconds": 8.588405191, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9377041529999985, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.588405191 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174433", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed/series.json b/results/gogs-moon/cadence-rolling-seed/series.json new file mode 100644 index 000000000..493eba22b --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "1fc9fdbfd692a7c3b1c3479e4c1cc1a1f55190e7a7f4a7f781986271e4a5cee8" +} diff --git a/results/helix-nix/cadence-fresh-01/completions/37320867908.json b/results/helix-nix/cadence-fresh-01/completions/37320867908.json new file mode 100644 index 000000000..b5e5c792b --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/completions/37320867908.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320867908", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320867908", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e103123da0dfd86047c1780a4ea86e37c4d55786742b3fb66d000554d9172fd0" +} diff --git a/results/helix-nix/cadence-fresh-01/completions/37325075463.json b/results/helix-nix/cadence-fresh-01/completions/37325075463.json new file mode 100644 index 000000000..ad890f297 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/completions/37325075463.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37325075463", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325075463", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "75038a56e31f991df749665d605a95fc42c71a9b68c90c9bd5545534c0df3a25" +} diff --git a/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json new file mode 100644 index 000000000..e6891e216 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:29:07Z", + "ref": "cadence-fresh-qualification", + "state": "requested", + "run_id": 37325075463, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325075463" +} diff --git a/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..ebe223dd2 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:28:34Z", + "ref": "shared-benchmark-cadence", + "state": "rejected", + "error": "GitHub HTTP 422: merged PR branch shared-benchmark-cadence was deleted; no ref found." +} diff --git a/results/helix-nix/cadence-fresh-01/report.json b/results/helix-nix/cadence-fresh-01/report.json new file mode 100644 index 000000000..4b1fca7ad --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/report.json @@ -0,0 +1,1486 @@ +{ + "schema_version": 1, + "plan_sha256": "c2a0ba9332802d28a66f0938b4fd31a43ff928f6d8e7348f2f07f37e1c933421", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320867908", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320867908", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e103123da0dfd86047c1780a4ea86e37c4d55786742b3fb66d000554d9172fd0" + }, + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37325075463", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325075463", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "75038a56e31f991df749665d605a95fc42c71a9b68c90c9bd5545534c0df3a25" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 281.6677286738425, + "min": 219.9609663066821, + "max": 343.3744910410028 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 7.060009012536909, + "min": 6.481956831083494, + "max": 7.638061193990325 + }, + "storage": { + "median": 46171613.0, + "min": 46171613, + "max": 46171613 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 2, + "measurement": { + "median": 367.90909346482397, + "min": 363.82386535294876, + "max": 371.99432157669924 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "cachix", + "count": 2, + "measurement": { + "median": 3.44525782967267, + "min": 3.2420969071586114, + "max": 3.6484187521867284 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.364945411682129, + "build_seconds": 215.59602089499998, + "workflow_seconds": null, + "total_seconds": 219.9609663066821, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 219.9609663066821 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174370", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.811720848083496, + "build_seconds": 0.6702359829999978, + "workflow_seconds": null, + "total_seconds": 6.481956831083494, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.481956831083494 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174502", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4744367599487305, + "build_seconds": 361.349428593, + "workflow_seconds": null, + "total_seconds": 363.82386535294876, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 363.82386535294876 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174369", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3420071601867676, + "build_seconds": 1.3064115919999608, + "workflow_seconds": null, + "total_seconds": 3.6484187521867284, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.6484187521867284 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174503", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.5731074810028076, + "build_seconds": 339.80138356, + "workflow_seconds": null, + "total_seconds": 343.3744910410028, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 343.3744910410028 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174628", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.2652809619903564, + "build_seconds": 0.37278023199996824, + "workflow_seconds": null, + "total_seconds": 7.638061193990325, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.638061193990325 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174648", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4242477416992188, + "build_seconds": 369.570073835, + "workflow_seconds": null, + "total_seconds": 371.99432157669924, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 371.99432157669924 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174627", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4607598781585693, + "build_seconds": 0.7813370290000421, + "workflow_seconds": null, + "total_seconds": 3.2420969071586114, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.2420969071586114 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174647", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/cadence-fresh-01/report.md b/results/helix-nix/cadence-fresh-01/report.md new file mode 100644 index 000000000..4048f4de0 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/report.md @@ -0,0 +1,33 @@ +# helix-nix: cadence-fresh-01 + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 2 | 281.6677286738425 | 219.9609663066821–343.3744910410028 | unmeasured | 0 | +| Warm build | BoringCache | 2 | 7.060009012536909 | 6.481956831083494–7.638061193990325 | 46171613.0 | 2 | +| Cold build | Cachix | 2 | 367.90909346482397 | 363.82386535294876–371.99432157669924 | unmeasured | 0 | +| Warm build | Cachix | 2 | 3.44525782967267 | 3.2420969071586114–3.6484187521867284 | unmeasured | 0 | + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 4.364945411682129 | 215.59602089499998 | 219.9609663066821 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 5.811720848083496 | 0.6702359829999978 | 6.481956831083494 | 46171613 | boringcache-check | hit | [JSON](runs/1-boringcache-warm.json) | +| 1 | Cachix | Cold build | 2.4744367599487305 | 361.349428593 | 363.82386535294876 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | +| 1 | Cachix | Warm build | 2.3420071601867676 | 1.3064115919999608 | 3.6484187521867284 | unmeasured | unmeasured | hit | [JSON](runs/1-cachix-warm.json) | +| 2 | BoringCache | Cold build | 3.5731074810028076 | 339.80138356 | 343.3744910410028 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 7.2652809619903564 | 0.37278023199996824 | 7.638061193990325 | 46171613 | boringcache-check | hit | [JSON](runs/2-boringcache-warm.json) | +| 2 | Cachix | Cold build | 2.4242477416992188 | 369.570073835 | 371.99432157669924 | unmeasured | unmeasured | miss | [JSON](runs/2-cachix-cold.json) | +| 2 | Cachix | Warm build | 2.4607598781585693 | 0.7813370290000421 | 3.2420969071586114 | unmeasured | unmeasured | hit | [JSON](runs/2-cachix-warm.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..fdb35e09a --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.364945411682129, + "build_seconds": 215.59602089499998, + "workflow_seconds": null, + "total_seconds": 219.9609663066821, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 219.9609663066821 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174370", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..aa31a95d6 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.811720848083496, + "build_seconds": 0.6702359829999978, + "workflow_seconds": null, + "total_seconds": 6.481956831083494, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.481956831083494 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174502", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json b/results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json new file mode 100644 index 000000000..3a47e8d5e --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4744367599487305, + "build_seconds": 361.349428593, + "workflow_seconds": null, + "total_seconds": 363.82386535294876, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 363.82386535294876 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174369", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json b/results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json new file mode 100644 index 000000000..42fc88da7 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3420071601867676, + "build_seconds": 1.3064115919999608, + "workflow_seconds": null, + "total_seconds": 3.6484187521867284, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.6484187521867284 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174503", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..88d564606 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.5731074810028076, + "build_seconds": 339.80138356, + "workflow_seconds": null, + "total_seconds": 343.3744910410028, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 343.3744910410028 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174628", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..0e0f908b8 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.2652809619903564, + "build_seconds": 0.37278023199996824, + "workflow_seconds": null, + "total_seconds": 7.638061193990325, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.638061193990325 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174648", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json b/results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json new file mode 100644 index 000000000..f24845015 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4242477416992188, + "build_seconds": 369.570073835, + "workflow_seconds": null, + "total_seconds": 371.99432157669924, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 371.99432157669924 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174627", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json b/results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json new file mode 100644 index 000000000..79226c16d --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4607598781585693, + "build_seconds": 0.7813370290000421, + "workflow_seconds": null, + "total_seconds": 3.2420969071586114, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.2420969071586114 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174647", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-rolling-advance/completions/37323117664.json b/results/helix-nix/cadence-rolling-advance/completions/37323117664.json new file mode 100644 index 000000000..bce8685e9 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/completions/37323117664.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323117664", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ccb6af059c8aae587960411129f7d4c7908c4555fe54f657899e18aa3e2b060" +} diff --git a/results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json b/results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..8785394b9 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:32Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323117664, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664" +} diff --git a/results/helix-nix/cadence-rolling-advance/report.json b/results/helix-nix/cadence-rolling-advance/report.json new file mode 100644 index 000000000..df30f8632 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/report.json @@ -0,0 +1,412 @@ +{ + "schema_version": 1, + "plan_sha256": "c1e34a971796d860ed2a472ecb705e2ee919c904b86ae87e3239d2dc8422241a", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323117664", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ccb6af059c8aae587960411129f7d4c7908c4555fe54f657899e18aa3e2b060" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 295.80389107529044, + "min": 295.80389107529044, + "max": 295.80389107529044 + }, + "storage": { + "median": 46171799, + "min": 46171799, + "max": 46171799 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 3.877758286099265, + "min": 3.877758286099265, + "max": 3.877758286099265 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6.215389013290405, + "build_seconds": 289.58850206200003, + "workflow_seconds": null, + "total_seconds": 295.80389107529044, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 295.80389107529044 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171799, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": true, + "measured_bytes": 46171799, + "unmeasured_tags": [], + "total_bytes": 46171799, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171799, + "compressed_size": 46171799, + "size": 46171799 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174619", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.420297861099243, + "build_seconds": 1.457460425000022, + "workflow_seconds": null, + "total_seconds": 3.877758286099265, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.877758286099265 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174620", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/cadence-rolling-advance/report.md b/results/helix-nix/cadence-rolling-advance/report.md new file mode 100644 index 000000000..42f482148 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/report.md @@ -0,0 +1,29 @@ +# helix-nix: cadence-rolling-advance + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 295.80389107529044 | 295.80389107529044–295.80389107529044 | 46171799 | 1 | +| Changed-source build | Cachix | 1 | 3.877758286099265 | 3.877758286099265–3.877758286099265 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 6.215389013290405 | 289.58850206200003 | 295.80389107529044 | 46171799 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.420297861099243 | 1.457460425000022 | 3.877758286099265 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..e0f483411 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6.215389013290405, + "build_seconds": 289.58850206200003, + "workflow_seconds": null, + "total_seconds": 295.80389107529044, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 295.80389107529044 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171799, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": true, + "measured_bytes": 46171799, + "unmeasured_tags": [], + "total_bytes": 46171799, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171799, + "compressed_size": 46171799, + "size": 46171799 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174619", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] +} diff --git a/results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json b/results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json new file mode 100644 index 000000000..bce549ccc --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.420297861099243, + "build_seconds": 1.457460425000022, + "workflow_seconds": null, + "total_seconds": 3.877758286099265, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.877758286099265 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174620", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] +} diff --git a/results/helix-nix/cadence-rolling-advance/series.json b/results/helix-nix/cadence-rolling-advance/series.json new file mode 100644 index 000000000..ef44c63d8 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "c1e34a971796d860ed2a472ecb705e2ee919c904b86ae87e3239d2dc8422241a" +} diff --git a/results/helix-nix/cadence-rolling-seed/completions/37322604229.json b/results/helix-nix/cadence-rolling-seed/completions/37322604229.json new file mode 100644 index 000000000..2c1533c2a --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/completions/37322604229.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322604229", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3f29172cdc9625dc804e52f16b92615b974db093eb00db3544ec28280cae465d" +} diff --git a/results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json b/results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..5268e6813 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322604229, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229" +} diff --git a/results/helix-nix/cadence-rolling-seed/report.json b/results/helix-nix/cadence-rolling-seed/report.json new file mode 100644 index 000000000..34d207f7b --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/report.json @@ -0,0 +1,407 @@ +{ + "schema_version": 1, + "plan_sha256": "76029a49e4208bb2c349ee7fdbca054c98b21b7afe37a050f9616d675d20002b", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322604229", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3f29172cdc9625dc804e52f16b92615b974db093eb00db3544ec28280cae465d" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 273.0119839951974, + "min": 273.0119839951974, + "max": 273.0119839951974 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 318.47876023020336, + "min": 318.47876023020336, + "max": 318.47876023020336 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1.9675590991973877, + "build_seconds": 271.044424896, + "workflow_seconds": null, + "total_seconds": 273.0119839951974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 273.0119839951974 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174509", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.734213352203369, + "build_seconds": 315.744546878, + "workflow_seconds": null, + "total_seconds": 318.47876023020336, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 318.47876023020336 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174508", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/cadence-rolling-seed/report.md b/results/helix-nix/cadence-rolling-seed/report.md new file mode 100644 index 000000000..9361ebcbb --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/report.md @@ -0,0 +1,29 @@ +# helix-nix: cadence-rolling-seed + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 273.0119839951974 | 273.0119839951974–273.0119839951974 | unmeasured | 0 | +| Changed-source build | Cachix | 1 | 318.47876023020336 | 318.47876023020336–318.47876023020336 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 1.9675590991973877 | 271.044424896 | 273.0119839951974 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.734213352203369 | 315.744546878 | 318.47876023020336 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..b4e844734 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1.9675590991973877, + "build_seconds": 271.044424896, + "workflow_seconds": null, + "total_seconds": 273.0119839951974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 273.0119839951974 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174509", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] +} diff --git a/results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json b/results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json new file mode 100644 index 000000000..13ff6dfc9 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.734213352203369, + "build_seconds": 315.744546878, + "workflow_seconds": null, + "total_seconds": 318.47876023020336, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 318.47876023020336 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174508", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] +} diff --git a/results/helix-nix/cadence-rolling-seed/series.json b/results/helix-nix/cadence-rolling-seed/series.json new file mode 100644 index 000000000..1b9262b32 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "76029a49e4208bb2c349ee7fdbca054c98b21b7afe37a050f9616d675d20002b" +} diff --git a/results/immich/cadence-rolling-advance/completions/37323121773.json b/results/immich/cadence-rolling-advance/completions/37323121773.json new file mode 100644 index 000000000..c66ea5ef4 --- /dev/null +++ b/results/immich/cadence-rolling-advance/completions/37323121773.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323121773", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f57898a2ccdc269f94623b46432c6b6280bfd6ef88b9288f990b9d630d852949" +} diff --git a/results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json b/results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..4471eb729 --- /dev/null +++ b/results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "immich", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:34Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323121773, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773" +} diff --git a/results/immich/cadence-rolling-advance/report.json b/results/immich/cadence-rolling-advance/report.json new file mode 100644 index 000000000..cac73388f --- /dev/null +++ b/results/immich/cadence-rolling-advance/report.json @@ -0,0 +1,445 @@ +{ + "schema_version": 1, + "plan_sha256": "02346d5e27c00699baa7c1376b045c7d89d732f05546d65a5be05de961ccb01a", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323121773", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f57898a2ccdc269f94623b46432c6b6280bfd6ef88b9288f990b9d630d852949" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 333, + "min": 333, + "max": 333 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 180, + "min": 180, + "max": 180 + }, + "storage": { + "median": 1750703493, + "min": 1750703493, + "max": 1750703493 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 328, + "workflow_seconds": 339, + "total_seconds": 333, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 333 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174468", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 185, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1750703493, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1750703493, + "unmeasured_tags": [], + "total_bytes": 1750703493, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "450988c6-cca4-460a-9667-b6274cc4970f", + "manifest_root_digest": "sha256:98e22c4f5630762cfa17505061947d0c6e87e52533bc629501c94922aebb3dfb", + "kv_total_size": 0, + "compressed_size": 1750703493, + "size": 1750703493 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174467", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] + } + ], + "exclusions": [] +} diff --git a/results/immich/cadence-rolling-advance/report.md b/results/immich/cadence-rolling-advance/report.md new file mode 100644 index 000000000..3082d959e --- /dev/null +++ b/results/immich/cadence-rolling-advance/report.md @@ -0,0 +1,29 @@ +# immich: cadence-rolling-advance + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 333 | 333–333 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 180 | 180–180 | 1750703493 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 328 | 333 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 180 | 180 | 1750703493 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json b/results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..151f7ee74 --- /dev/null +++ b/results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json @@ -0,0 +1,167 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 328, + "workflow_seconds": 339, + "total_seconds": 333, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 333 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174468", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] +} diff --git a/results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..9571a4fb6 --- /dev/null +++ b/results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,213 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 185, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1750703493, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1750703493, + "unmeasured_tags": [], + "total_bytes": 1750703493, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "450988c6-cca4-460a-9667-b6274cc4970f", + "manifest_root_digest": "sha256:98e22c4f5630762cfa17505061947d0c6e87e52533bc629501c94922aebb3dfb", + "kv_total_size": 0, + "compressed_size": 1750703493, + "size": 1750703493 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174467", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] +} diff --git a/results/immich/cadence-rolling-advance/series.json b/results/immich/cadence-rolling-advance/series.json new file mode 100644 index 000000000..d82c43eed --- /dev/null +++ b/results/immich/cadence-rolling-advance/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "variant": null, + "workflow_inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "02346d5e27c00699baa7c1376b045c7d89d732f05546d65a5be05de961ccb01a" +} diff --git a/results/immich/cadence-rolling-seed/completions/37321739971.json b/results/immich/cadence-rolling-seed/completions/37321739971.json new file mode 100644 index 000000000..85c510141 --- /dev/null +++ b/results/immich/cadence-rolling-seed/completions/37321739971.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37321739971", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "07eb4d0fc8d5073ad36895d85586e3b25a17b6d8671c966903a76bacd3f7f27d" +} diff --git a/results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json b/results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0b0385ad3 --- /dev/null +++ b/results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "immich", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:04:15Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321739971, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971" +} diff --git a/results/immich/cadence-rolling-seed/report.json b/results/immich/cadence-rolling-seed/report.json new file mode 100644 index 000000000..c256a1de7 --- /dev/null +++ b/results/immich/cadence-rolling-seed/report.json @@ -0,0 +1,445 @@ +{ + "schema_version": 1, + "plan_sha256": "18841129bbf62c7426a8451343792d2b8276bb379fc46fc6666e01f6666b7817", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37321739971", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "07eb4d0fc8d5073ad36895d85586e3b25a17b6d8671c966903a76bacd3f7f27d" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 364, + "min": 364, + "max": 364 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 187, + "min": 187, + "max": 187 + }, + "storage": { + "median": 1756603878, + "min": 1756603878, + "max": 1756603878 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 352, + "workflow_seconds": 370, + "total_seconds": 364, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 364 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174347", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 187, + "workflow_seconds": 194, + "total_seconds": 187, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 187 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1756603878, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1756603878, + "unmeasured_tags": [], + "total_bytes": 1756603878, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "30ebce4e-787e-4a47-8739-c431adc97c86", + "manifest_root_digest": "sha256:1f8ec2caa2839792df37e0036ea0a297c1df98fac25d22daa13b5221b97ce453", + "kv_total_size": 0, + "compressed_size": 1756603878, + "size": 1756603878 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174348", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] + } + ], + "exclusions": [] +} diff --git a/results/immich/cadence-rolling-seed/report.md b/results/immich/cadence-rolling-seed/report.md new file mode 100644 index 000000000..55eea9257 --- /dev/null +++ b/results/immich/cadence-rolling-seed/report.md @@ -0,0 +1,29 @@ +# immich: cadence-rolling-seed + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 364 | 364–364 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 187 | 187–187 | 1756603878 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 12 | 352 | 364 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 187 | 187 | 1756603878 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json b/results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..d25926bd6 --- /dev/null +++ b/results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json @@ -0,0 +1,167 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 352, + "workflow_seconds": 370, + "total_seconds": 364, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 364 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174347", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] +} diff --git a/results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d4a6e2efc --- /dev/null +++ b/results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,213 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 187, + "workflow_seconds": 194, + "total_seconds": 187, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 187 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1756603878, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1756603878, + "unmeasured_tags": [], + "total_bytes": 1756603878, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "30ebce4e-787e-4a47-8739-c431adc97c86", + "manifest_root_digest": "sha256:1f8ec2caa2839792df37e0036ea0a297c1df98fac25d22daa13b5221b97ce453", + "kv_total_size": 0, + "compressed_size": 1756603878, + "size": 1756603878 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174348", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] +} diff --git a/results/immich/cadence-rolling-seed/series.json b/results/immich/cadence-rolling-seed/series.json new file mode 100644 index 000000000..5d799c1c1 --- /dev/null +++ b/results/immich/cadence-rolling-seed/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "variant": null, + "workflow_inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "18841129bbf62c7426a8451343792d2b8276bb379fc46fc6666e01f6666b7817" +} diff --git a/results/mastodon/cadence-rolling-advance-server/completions/37323126507.json b/results/mastodon/cadence-rolling-advance-server/completions/37323126507.json new file mode 100644 index 000000000..6ad771b1a --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/completions/37323126507.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-server", + "sample": 1, + "run_id": "37323126507", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c7cdaec26745cefee9e346dba7b3ca2260869793a5a43790bb7c2919576e8bc" +} diff --git a/results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0c5f3a0cf --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server", + "series_id": "cadence-rolling-advance-server", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:36Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323126507, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507" +} diff --git a/results/mastodon/cadence-rolling-advance-server/report.json b/results/mastodon/cadence-rolling-advance-server/report.json new file mode 100644 index 000000000..721a2fdfb --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "94d5db9803d90b2f237a3d998a5c6cf43cbd12ad3cdb27c6ff8c897e43299d40", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-server", + "sample": 1, + "run_id": "37323126507", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c7cdaec26745cefee9e346dba7b3ca2260869793a5a43790bb7c2919576e8bc" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 299, + "min": 299, + "max": 299 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 180, + "min": 180, + "max": 180 + }, + "storage": { + "median": 1131047375, + "min": 1131047375, + "max": 1131047375 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 293, + "workflow_seconds": 308, + "total_seconds": 299, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 299 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174472", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 187, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1131047375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1131047375, + "unmeasured_tags": [], + "total_bytes": 1131047375, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "bb8a82c2-185a-45ab-a182-59a2bab44ac2", + "manifest_root_digest": "sha256:f39b48d505b211e9e0a5be7430c8e0484cb1677ca5f39c6039c81897dcff5176", + "kv_total_size": 0, + "compressed_size": 1131047375, + "size": 1131047375 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174471", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-advance-server/report.md b/results/mastodon/cadence-rolling-advance-server/report.md new file mode 100644 index 000000000..c02aa05c3 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-advance-server + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 299 | 299–299 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 180 | 180–180 | 1131047375 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 6 | 293 | 299 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 180 | 180 | 1131047375 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..310464ca2 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 293, + "workflow_seconds": 308, + "total_seconds": 299, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 299 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174472", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json new file mode 100644 index 000000000..63f2fa0fa --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 187, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1131047375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1131047375, + "unmeasured_tags": [], + "total_bytes": 1131047375, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "bb8a82c2-185a-45ab-a182-59a2bab44ac2", + "manifest_root_digest": "sha256:f39b48d505b211e9e0a5be7430c8e0484cb1677ca5f39c6039c81897dcff5176", + "kv_total_size": 0, + "compressed_size": 1131047375, + "size": 1131047375 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174471", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-server/series.json b/results/mastodon/cadence-rolling-advance-server/series.json new file mode 100644 index 000000000..a86c5f335 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-server", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "variant": "server", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "94d5db9803d90b2f237a3d998a5c6cf43cbd12ad3cdb27c6ff8c897e43299d40" +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json b/results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json new file mode 100644 index 000000000..e21bd88d8 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-streaming", + "sample": 1, + "run_id": "37323131290", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "acb6f806ef1e53607119c0ef8efcd7512dcd83ba7ed89c2b9940e5cacaf32884" +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..cadc4eef7 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming", + "series_id": "cadence-rolling-advance-streaming", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323131290, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290" +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/report.json b/results/mastodon/cadence-rolling-advance-streaming/report.json new file mode 100644 index 000000000..68fc7d95a --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "a5828cd7050d1f29ba9eecff83d2661ac333582448b631d428141d83d99b16f0", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-streaming", + "sample": 1, + "run_id": "37323131290", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "acb6f806ef1e53607119c0ef8efcd7512dcd83ba7ed89c2b9940e5cacaf32884" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 44, + "min": 44, + "max": 44 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 23, + "min": 23, + "max": 23 + }, + "storage": { + "median": 106316055, + "min": 106316055, + "max": 106316055 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 21, + "build_seconds": 23, + "workflow_seconds": 48, + "total_seconds": 44, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 44 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174473", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 23, + "workflow_seconds": 29, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106316055, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106316055, + "unmeasured_tags": [], + "total_bytes": 106316055, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "08c60a15-9c1d-4d96-a20a-3466790ea1b2", + "manifest_root_digest": "sha256:d7e2d159bd05e94b274c139460296fb261ed19046a9f321f6b07b4570ff99cc6", + "kv_total_size": 0, + "compressed_size": 106316055, + "size": 106316055 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174474", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/report.md b/results/mastodon/cadence-rolling-advance-streaming/report.md new file mode 100644 index 000000000..779eb1b65 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-advance-streaming + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 44 | 44–44 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 23 | 23–23 | 106316055 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 21 | 23 | 44 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 23 | 23 | 106316055 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..a6fdf6583 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 21, + "build_seconds": 23, + "workflow_seconds": 48, + "total_seconds": 44, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 44 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174473", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d7ace5173 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 23, + "workflow_seconds": 29, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106316055, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106316055, + "unmeasured_tags": [], + "total_bytes": 106316055, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "08c60a15-9c1d-4d96-a20a-3466790ea1b2", + "manifest_root_digest": "sha256:d7e2d159bd05e94b274c139460296fb261ed19046a9f321f6b07b4570ff99cc6", + "kv_total_size": 0, + "compressed_size": 106316055, + "size": 106316055 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174474", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/series.json b/results/mastodon/cadence-rolling-advance-streaming/series.json new file mode 100644 index 000000000..dbc01391c --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-streaming", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "variant": "streaming", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "a5828cd7050d1f29ba9eecff83d2661ac333582448b631d428141d83d99b16f0" +} diff --git a/results/mastodon/cadence-rolling-seed-server/completions/37321744434.json b/results/mastodon/cadence-rolling-seed-server/completions/37321744434.json new file mode 100644 index 000000000..e4b3d5fbe --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/completions/37321744434.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-server", + "sample": 1, + "run_id": "37321744434", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a548877f8cf73415bd6153d723a27a1d1c008a24881391acb25ac4a8338c763a" +} diff --git a/results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..d54dd94ed --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server", + "series_id": "cadence-rolling-seed-server", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:18Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321744434, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434" +} diff --git a/results/mastodon/cadence-rolling-seed-server/report.json b/results/mastodon/cadence-rolling-seed-server/report.json new file mode 100644 index 000000000..ec570194a --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "561a3c72c845a65485bb0a53dfd0e2be0baa59ac65533a44505140b8fac8d7c6", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-server", + "sample": 1, + "run_id": "37321744434", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a548877f8cf73415bd6153d723a27a1d1c008a24881391acb25ac4a8338c763a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 493, + "min": 493, + "max": 493 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 529, + "min": 529, + "max": 529 + }, + "storage": { + "median": 1130441371, + "min": 1130441371, + "max": 1130441371 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 484, + "workflow_seconds": 500, + "total_seconds": 493, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 493 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174351", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 529, + "workflow_seconds": 538, + "total_seconds": 529, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 529 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130441371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130441371, + "unmeasured_tags": [], + "total_bytes": 1130441371, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "10417425-d059-41fe-81d0-c0f00e9a00ac", + "manifest_root_digest": "sha256:09d1b510600ce403588247bd81f36109c76272d0192fe149c873aa723961e238", + "kv_total_size": 0, + "compressed_size": 1130441371, + "size": 1130441371 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174352", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-seed-server/report.md b/results/mastodon/cadence-rolling-seed-server/report.md new file mode 100644 index 000000000..39c072572 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-seed-server + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 493 | 493–493 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 529 | 529–529 | 1130441371 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 9 | 484 | 493 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 529 | 529 | 1130441371 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..6f2426b2f --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 484, + "workflow_seconds": 500, + "total_seconds": 493, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 493 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174351", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json new file mode 100644 index 000000000..62b22dafc --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 529, + "workflow_seconds": 538, + "total_seconds": 529, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 529 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130441371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130441371, + "unmeasured_tags": [], + "total_bytes": 1130441371, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "10417425-d059-41fe-81d0-c0f00e9a00ac", + "manifest_root_digest": "sha256:09d1b510600ce403588247bd81f36109c76272d0192fe149c873aa723961e238", + "kv_total_size": 0, + "compressed_size": 1130441371, + "size": 1130441371 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174352", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-server/series.json b/results/mastodon/cadence-rolling-seed-server/series.json new file mode 100644 index 000000000..d443f32f6 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-server", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "variant": "server", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "561a3c72c845a65485bb0a53dfd0e2be0baa59ac65533a44505140b8fac8d7c6" +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json b/results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json new file mode 100644 index 000000000..46f92f822 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-streaming", + "sample": 1, + "run_id": "37321748690", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a1f7eca3d823149f094a2316b6c4310d255e6f71752359e90b344b8e55ed88c" +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..50eaa1e3a --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming", + "series_id": "cadence-rolling-seed-streaming", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:20Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321748690, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690" +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/report.json b/results/mastodon/cadence-rolling-seed-streaming/report.json new file mode 100644 index 000000000..203e0e399 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "f673553fdf484d6f3aadc8ecaaa224a32e1985da51fbe20076919e043bb5e6ca", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-streaming", + "sample": 1, + "run_id": "37321748690", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a1f7eca3d823149f094a2316b6c4310d255e6f71752359e90b344b8e55ed88c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 32, + "min": 32, + "max": 32 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 27, + "min": 27, + "max": 27 + }, + "storage": { + "median": 106284886, + "min": 106284886, + "max": 106284886 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 24, + "workflow_seconds": 39, + "total_seconds": 32, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 32 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174354", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 27, + "workflow_seconds": 35, + "total_seconds": 27, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 27 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106284886, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106284886, + "unmeasured_tags": [], + "total_bytes": 106284886, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eac04b9-8b45-49eb-9791-3d3296be20b7", + "manifest_root_digest": "sha256:57a408410d24dfd969eef8ffd6dbf28ec28a8099c8cf6d471558a98a333ad2aa", + "kv_total_size": 0, + "compressed_size": 106284886, + "size": 106284886 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174355", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/report.md b/results/mastodon/cadence-rolling-seed-streaming/report.md new file mode 100644 index 000000000..d23f27b4e --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-seed-streaming + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 32 | 32–32 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 27 | 27–27 | 106284886 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 8 | 24 | 32 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 27 | 27 | 106284886 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..f1ef2cc19 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 24, + "workflow_seconds": 39, + "total_seconds": 32, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 32 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174354", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json new file mode 100644 index 000000000..332bc30da --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 27, + "workflow_seconds": 35, + "total_seconds": 27, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 27 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106284886, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106284886, + "unmeasured_tags": [], + "total_bytes": 106284886, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eac04b9-8b45-49eb-9791-3d3296be20b7", + "manifest_root_digest": "sha256:57a408410d24dfd969eef8ffd6dbf28ec28a8099c8cf6d471558a98a333ad2aa", + "kv_total_size": 0, + "compressed_size": 106284886, + "size": 106284886 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174355", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/series.json b/results/mastodon/cadence-rolling-seed-streaming/series.json new file mode 100644 index 000000000..6a532306d --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-streaming", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "variant": "streaming", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "f673553fdf484d6f3aadc8ecaaa224a32e1985da51fbe20076919e043bb5e6ca" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json b/results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json new file mode 100644 index 000000000..93c7c9a52 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324214890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a49c2ba2066ca33e2fef110f6cf68bbff938c9ba3212571c8dc6aaf97a99799d" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..fc0377830 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:43Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324214890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/report.json b/results/msgpack-sbt/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..88f52275a --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "87754745e5a804e3a837535e69b5636562773f538f358d17eb5631502268ac20", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324214890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a49c2ba2066ca33e2fef110f6cf68bbff938c9ba3212571c8dc6aaf97a99799d" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10113066600000309, + "build_seconds": 22.980055022000002, + "workflow_seconds": null, + "total_seconds": 23.081185688000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013612939999916307, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 23.081185688000005 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174559", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20128656899998987, + "build_seconds": 25.342844542999984, + "workflow_seconds": null, + "total_seconds": 25.544131111999974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7568644210000173, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.544131111999974 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174560", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/report.md b/results/msgpack-sbt/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..b584a9f04 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/report.md @@ -0,0 +1,30 @@ +# msgpack-sbt: cadence-rolling-advance-02 + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10113066600000309 | 22.980055022000002 | 23.081185688000005 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20128656899998987 | 25.342844542999984 | 25.544131111999974 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..19cd9c3de --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10113066600000309, + "build_seconds": 22.980055022000002, + "workflow_seconds": null, + "total_seconds": 23.081185688000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013612939999916307, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 23.081185688000005 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174559", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..850147393 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20128656899998987, + "build_seconds": 25.342844542999984, + "workflow_seconds": null, + "total_seconds": 25.544131111999974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7568644210000173, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.544131111999974 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174560", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/series.json b/results/msgpack-sbt/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..ee288bf74 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "87754745e5a804e3a837535e69b5636562773f538f358d17eb5631502268ac20" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json b/results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json new file mode 100644 index 000000000..154163913 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323136088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote msgpack-sbt commit: failure", + "msgpack-sbt rolling report: failure" + ], + "evidence_manifest_sha256": "9efdf7335b4808dd3a3bb626666cc167473699164c141d035c288bda6127701f" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..b199e1c8d --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323136088, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/report.json b/results/msgpack-sbt/cadence-rolling-advance/report.json new file mode 100644 index 000000000..12b36cf1a --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/report.json @@ -0,0 +1,223 @@ +{ + "schema_version": 1, + "plan_sha256": "cd2354c80ad3f6fa4d0543772b2ced703b776abc0cdadab4a02aa9d4889483e6", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323136088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote msgpack-sbt commit: failure", + "msgpack-sbt rolling report: failure" + ], + "evidence_manifest_sha256": "9efdf7335b4808dd3a3bb626666cc167473699164c141d035c288bda6127701f" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323136088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote msgpack-sbt commit: failure", + "msgpack-sbt rolling report: failure" + ], + "evidence_manifest_sha256": "9efdf7335b4808dd3a3bb626666cc167473699164c141d035c288bda6127701f" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "80307420630483e2c47545a5deeca56f262b5772eb594db79cde6c03b2c477b2", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10089654600000131, + "build_seconds": 24.936086543999977, + "workflow_seconds": null, + "total_seconds": 25.03698308999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3992640499999993, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.03698308999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323136088", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174481", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323136088-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323136088" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/report.md b/results/msgpack-sbt/cadence-rolling-advance/report.md new file mode 100644 index 000000000..89c70881b --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# msgpack-sbt: cadence-rolling-advance + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323136088 failed completion checks: + +- Workflow concluded failure +- bazel-remote msgpack-sbt commit: failure +- msgpack-sbt rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.10089654600000131 | 24.936086543999977 | 25.03698308999998 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6550e89e7 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "80307420630483e2c47545a5deeca56f262b5772eb594db79cde6c03b2c477b2", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10089654600000131, + "build_seconds": 24.936086543999977, + "workflow_seconds": null, + "total_seconds": 25.03698308999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3992640499999993, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.03698308999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323136088", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174481", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323136088-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323136088" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/series.json b/results/msgpack-sbt/cadence-rolling-advance/series.json new file mode 100644 index 000000000..2c1a05853 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "80307420630483e2c47545a5deeca56f262b5772eb594db79cde6c03b2c477b2", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "cd2354c80ad3f6fa4d0543772b2ced703b776abc0cdadab4a02aa9d4889483e6" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json b/results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json new file mode 100644 index 000000000..baacb0813 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324036688", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3171de3a2ed83c157c5fd6780148832cff76ae26a9dca46d962c45e13fb00020" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..8fe011d54 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:24Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324036688, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/report.json b/results/msgpack-sbt/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..1e2756f84 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "b1568aacb3510adfb902817b524a2ea72592c56a4953b7d78320f182d14fdb98", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324036688", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3171de3a2ed83c157c5fd6780148832cff76ae26a9dca46d962c45e13fb00020" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108540399998844, + "build_seconds": 38.53932476999999, + "workflow_seconds": null, + "total_seconds": 38.64041017399998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016968049999945833, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.64041017399998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174535", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10086146499999415, + "build_seconds": 27.882007873000006, + "workflow_seconds": null, + "total_seconds": 27.982869338, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4697909409999994, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.982869338 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174536", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/report.md b/results/msgpack-sbt/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..c9473200e --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# msgpack-sbt: cadence-rolling-seed-02 + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10108540399998844 | 38.53932476999999 | 38.64041017399998 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.10086146499999415 | 27.882007873000006 | 27.982869338 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..568f1136c --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108540399998844, + "build_seconds": 38.53932476999999, + "workflow_seconds": null, + "total_seconds": 38.64041017399998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016968049999945833, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.64041017399998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174535", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..aeefd9264 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10086146499999415, + "build_seconds": 27.882007873000006, + "workflow_seconds": null, + "total_seconds": 27.982869338, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4697909409999994, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.982869338 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174536", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/series.json b/results/msgpack-sbt/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..82aef726c --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:12Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b1568aacb3510adfb902817b524a2ea72592c56a4953b7d78320f182d14fdb98" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json b/results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json new file mode 100644 index 000000000..09b003459 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322609279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322609279", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ee073a8869a31c3560df15b0cde8acc75ddd48d2efffadc707b7ba8da2e49fd7" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..fac4759d9 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:43Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322609279, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322609279" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/report.json b/results/msgpack-sbt/cadence-rolling-seed/report.json new file mode 100644 index 000000000..67640dbeb --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "e563c124fae3f2f6a44f5c7921ea7dfc79ded103d502dd402e643463bab6cde7", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322609279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322609279", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ee073a8869a31c3560df15b0cde8acc75ddd48d2efffadc707b7ba8da2e49fd7" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111915200000965, + "build_seconds": 39.54074584200001, + "workflow_seconds": null, + "total_seconds": 39.64186499400002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001766992999989725, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 39.64186499400002 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174442", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20117491299998846, + "build_seconds": 38.363812155000005, + "workflow_seconds": null, + "total_seconds": 38.56498706799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6816028860000074, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.56498706799999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174441", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/report.md b/results/msgpack-sbt/cadence-rolling-seed/report.md new file mode 100644 index 000000000..9f1865671 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# msgpack-sbt: cadence-rolling-seed + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10111915200000965 | 39.54074584200001 | 39.64186499400002 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20117491299998846 | 38.363812155000005 | 38.56498706799999 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6c6e7fa49 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111915200000965, + "build_seconds": 39.54074584200001, + "workflow_seconds": null, + "total_seconds": 39.64186499400002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001766992999989725, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 39.64186499400002 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174442", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..661540de1 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20117491299998846, + "build_seconds": 38.363812155000005, + "workflow_seconds": null, + "total_seconds": 38.56498706799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6816028860000074, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.56498706799999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174441", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/series.json b/results/msgpack-sbt/cadence-rolling-seed/series.json new file mode 100644 index 000000000..0afee4648 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "e563c124fae3f2f6a44f5c7921ea7dfc79ded103d502dd402e643463bab6cde7" +} diff --git a/results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json b/results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json new file mode 100644 index 000000000..e5d20102c --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-distroless", + "sample": 1, + "run_id": "37323140444", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e400914e058065b703dd4877a9bc49d2827c428e0a1c6b68c4f0a512fa8b3487" +} diff --git a/results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..ae0e4e952 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless", + "series_id": "cadence-rolling-advance-distroless", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:43Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323140444, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444" +} diff --git a/results/n8n/cadence-rolling-advance-distroless/report.json b/results/n8n/cadence-rolling-advance-distroless/report.json new file mode 100644 index 000000000..7ea6f0ee4 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "81a8042363d108962966347f75d613c85f5c624d58fcbec68a25b3563c1a2082", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-distroless", + "sample": 1, + "run_id": "37323140444", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e400914e058065b703dd4877a9bc49d2827c428e0a1c6b68c4f0a512fa8b3487" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 128, + "min": 128, + "max": 128 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 87, + "min": 87, + "max": 87 + }, + "storage": { + "median": 581233979, + "min": 581233979, + "max": 581233979 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 119, + "workflow_seconds": 285, + "total_seconds": 128, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 128 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174476", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 87, + "workflow_seconds": 249, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581233979, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581233979, + "unmeasured_tags": [], + "total_bytes": 581233979, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "1c175ac8-a90f-48fc-8ce3-65c683d162ed", + "manifest_root_digest": "sha256:24200a6466d459f8679011ec2e110e39bef6ef8901722551303446c3b13fda40", + "kv_total_size": 0, + "compressed_size": 581233979, + "size": 581233979 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174477", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-distroless/report.md b/results/n8n/cadence-rolling-advance-distroless/report.md new file mode 100644 index 000000000..66c68035c --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-distroless + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 128 | 128–128 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 87 | 87–87 | 581233979 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 9 | 119 | 128 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 87 | 87 | 581233979 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..e44eecc41 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 119, + "workflow_seconds": 285, + "total_seconds": 128, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 128 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174476", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] +} diff --git a/results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6125cade9 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 87, + "workflow_seconds": 249, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581233979, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581233979, + "unmeasured_tags": [], + "total_bytes": 581233979, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "1c175ac8-a90f-48fc-8ce3-65c683d162ed", + "manifest_root_digest": "sha256:24200a6466d459f8679011ec2e110e39bef6ef8901722551303446c3b13fda40", + "kv_total_size": 0, + "compressed_size": 581233979, + "size": 581233979 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174477", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] +} diff --git a/results/n8n/cadence-rolling-advance-distroless/series.json b/results/n8n/cadence-rolling-advance-distroless/series.json new file mode 100644 index 000000000..9114f45fb --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-distroless", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "distroless", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "81a8042363d108962966347f75d613c85f5c624d58fcbec68a25b3563c1a2082" +} diff --git a/results/n8n/cadence-rolling-advance-docker/completions/37323145174.json b/results/n8n/cadence-rolling-advance-docker/completions/37323145174.json new file mode 100644 index 000000000..8ffe329a1 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/completions/37323145174.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-docker", + "sample": 1, + "run_id": "37323145174", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ad802f0c8845d5f11e782410c92ba8ef0ac2ac46866b617108445404349e986" +} diff --git a/results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..50ad29f00 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker", + "series_id": "cadence-rolling-advance-docker", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:45Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323145174, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174" +} diff --git a/results/n8n/cadence-rolling-advance-docker/report.json b/results/n8n/cadence-rolling-advance-docker/report.json new file mode 100644 index 000000000..078997ef7 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "85ee37fee93a651e47f12c7a5fff50734381e919b02763f2c371359baa239de3", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-docker", + "sample": 1, + "run_id": "37323145174", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ad802f0c8845d5f11e782410c92ba8ef0ac2ac46866b617108445404349e986" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 178, + "min": 178, + "max": 178 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 152, + "min": 152, + "max": 152 + }, + "storage": { + "median": 483880710, + "min": 483880710, + "max": 483880710 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 173, + "workflow_seconds": 384, + "total_seconds": 178, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 178 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174488", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 152, + "workflow_seconds": 351, + "total_seconds": 152, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 152 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483880710, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483880710, + "unmeasured_tags": [], + "total_bytes": 483880710, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "68ec16be-5a23-408a-aaef-3d4cdcd6d219", + "manifest_root_digest": "sha256:340b591e8568dcd06731dc4a0585f64e5c735a7da2b952f6eb31614e1190c243", + "kv_total_size": 0, + "compressed_size": 483880710, + "size": 483880710 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174489", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-docker/report.md b/results/n8n/cadence-rolling-advance-docker/report.md new file mode 100644 index 000000000..be60ebd00 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-docker + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 178 | 178–178 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 152 | 152–152 | 483880710 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 173 | 178 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 152 | 152 | 483880710 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..89257230f --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 173, + "workflow_seconds": 384, + "total_seconds": 178, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 178 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174488", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] +} diff --git a/results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6b4ff6239 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 152, + "workflow_seconds": 351, + "total_seconds": 152, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 152 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483880710, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483880710, + "unmeasured_tags": [], + "total_bytes": 483880710, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "68ec16be-5a23-408a-aaef-3d4cdcd6d219", + "manifest_root_digest": "sha256:340b591e8568dcd06731dc4a0585f64e5c735a7da2b952f6eb31614e1190c243", + "kv_total_size": 0, + "compressed_size": 483880710, + "size": 483880710 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174489", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] +} diff --git a/results/n8n/cadence-rolling-advance-docker/series.json b/results/n8n/cadence-rolling-advance-docker/series.json new file mode 100644 index 000000000..557e3ae0f --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-docker", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "docker", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "85ee37fee93a651e47f12c7a5fff50734381e919b02763f2c371359baa239de3" +} diff --git a/results/n8n/cadence-rolling-advance-runners/completions/37323150431.json b/results/n8n/cadence-rolling-advance-runners/completions/37323150431.json new file mode 100644 index 000000000..758123266 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/completions/37323150431.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-runners", + "sample": 1, + "run_id": "37323150431", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "9b87465fe9de14a641423ad22393b66514278972c78f5aab173864f341c67a5f" +} diff --git a/results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..bcd69f9b8 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners", + "series_id": "cadence-rolling-advance-runners", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:47Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323150431, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431" +} diff --git a/results/n8n/cadence-rolling-advance-runners/report.json b/results/n8n/cadence-rolling-advance-runners/report.json new file mode 100644 index 000000000..130d20deb --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "e7626673b7f2595136f07f3f3953bffe5f2e1d64ecf8b9775519da47bf0aa605", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-runners", + "sample": 1, + "run_id": "37323150431", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "9b87465fe9de14a641423ad22393b66514278972c78f5aab173864f341c67a5f" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 108, + "min": 108, + "max": 108 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 59, + "min": 59, + "max": 59 + }, + "storage": { + "median": 413978377, + "min": 413978377, + "max": 413978377 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 98, + "workflow_seconds": 308, + "total_seconds": 108, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 108 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174478", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 59, + "workflow_seconds": 256, + "total_seconds": 59, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 59 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413978377, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413978377, + "unmeasured_tags": [], + "total_bytes": 413978377, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9e8e1ca5-b03b-4a29-9314-be454f6d3877", + "manifest_root_digest": "sha256:529ae2cf71c27d48484f8415c6e120b65d63caabbf808d1d1b48e18dd006c741", + "kv_total_size": 0, + "compressed_size": 413978377, + "size": 413978377 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174479", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-runners/report.md b/results/n8n/cadence-rolling-advance-runners/report.md new file mode 100644 index 000000000..c25560fb1 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-runners + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 108 | 108–108 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 59 | 59–59 | 413978377 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 10 | 98 | 108 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 59 | 59 | 413978377 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..ca7dbf9ed --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 98, + "workflow_seconds": 308, + "total_seconds": 108, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 108 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174478", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] +} diff --git a/results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json new file mode 100644 index 000000000..c0b350957 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 59, + "workflow_seconds": 256, + "total_seconds": 59, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 59 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413978377, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413978377, + "unmeasured_tags": [], + "total_bytes": 413978377, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9e8e1ca5-b03b-4a29-9314-be454f6d3877", + "manifest_root_digest": "sha256:529ae2cf71c27d48484f8415c6e120b65d63caabbf808d1d1b48e18dd006c741", + "kv_total_size": 0, + "compressed_size": 413978377, + "size": 413978377 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174479", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] +} diff --git a/results/n8n/cadence-rolling-advance-runners/series.json b/results/n8n/cadence-rolling-advance-runners/series.json new file mode 100644 index 000000000..d247d0e8a --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-runners", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "runners", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "e7626673b7f2595136f07f3f3953bffe5f2e1d64ecf8b9775519da47bf0aa605" +} diff --git a/results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json b/results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json new file mode 100644 index 000000000..f12a3eaef --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-turbo", + "sample": 1, + "run_id": "37323155106", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e0dcca70dfbc76d3d4ea52feccbdaf0c6a47ad2111e0432ca3c184010a6920f9" +} diff --git a/results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..5e3702c7b --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo", + "series_id": "cadence-rolling-advance-turbo", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:49Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323155106, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106" +} diff --git a/results/n8n/cadence-rolling-advance-turbo/report.json b/results/n8n/cadence-rolling-advance-turbo/report.json new file mode 100644 index 000000000..7be9956eb --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/report.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "plan_sha256": "b4b90237fb539c9315406c5e54501cb951d86de26a04509e10b3086b20bc4e24", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-turbo", + "sample": 1, + "run_id": "37323155106", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e0dcca70dfbc76d3d4ea52feccbdaf0c6a47ad2111e0432ca3c184010a6920f9" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 134, + "min": 134, + "max": 134 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 131, + "min": 131, + "max": 131 + }, + "storage": { + "median": 96876410, + "min": 96876410, + "max": 96876410 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 133, + "workflow_seconds": 170, + "total_seconds": 134, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 134 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-206fc6f317e353a98f7359801376a22d6bd136a2", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174482", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 124, + "workflow_seconds": 166, + "total_seconds": 131, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 131 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 96876410, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 96876410, + "unmeasured_tags": [], + "total_bytes": 96876410, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 96876410, + "compressed_size": 96876410, + "size": 96876410 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174483", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-turbo/report.md b/results/n8n/cadence-rolling-advance-turbo/report.md new file mode 100644 index 000000000..fc032cdf5 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-turbo + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 134 | 134–134 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 131 | 131–131 | 96876410 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 1 | 133 | 134 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 7 | 124 | 131 | 96876410 | boringcache-check | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..2ade4d40f --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 133, + "workflow_seconds": 170, + "total_seconds": 134, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 134 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-206fc6f317e353a98f7359801376a22d6bd136a2", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174482", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] +} diff --git a/results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6c38b5815 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 124, + "workflow_seconds": 166, + "total_seconds": 131, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 131 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 96876410, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 96876410, + "unmeasured_tags": [], + "total_bytes": 96876410, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 96876410, + "compressed_size": 96876410, + "size": 96876410 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174483", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] +} diff --git a/results/n8n/cadence-rolling-advance-turbo/series.json b/results/n8n/cadence-rolling-advance-turbo/series.json new file mode 100644 index 000000000..1859839ab --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-turbo", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "b4b90237fb539c9315406c5e54501cb951d86de26a04509e10b3086b20bc4e24" +} diff --git a/results/n8n/cadence-rolling-next-turbo/completions/37325183094.json b/results/n8n/cadence-rolling-next-turbo/completions/37325183094.json new file mode 100644 index 000000000..f6b0597e7 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/completions/37325183094.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-next-turbo", + "sample": 1, + "run_id": "37325183094", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094", + "source_sha": "173fbed3116e31cad526e2e4d777f3f8594c47ce", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1a7d6899fa0e314c6a68699f3312ef3c9e51b950b3a62522960b1e371057e50" +} diff --git a/results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0a8ba9426 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo", + "series_id": "cadence-rolling-next-turbo", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:29:56Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325183094, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094" +} diff --git a/results/n8n/cadence-rolling-next-turbo/report.json b/results/n8n/cadence-rolling-next-turbo/report.json new file mode 100644 index 000000000..33c3d89a9 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/report.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "plan_sha256": "b749a682eb42d6c7a423da19c572b9ccd92b020e2c0b11b072e4da3d6a1884c1", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-next-turbo", + "sample": 1, + "run_id": "37325183094", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094", + "source_sha": "173fbed3116e31cad526e2e4d777f3f8594c47ce", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1a7d6899fa0e314c6a68699f3312ef3c9e51b950b3a62522960b1e371057e50" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 23, + "min": 23, + "max": 23 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": { + "median": 102947794, + "min": 102947794, + "max": 102947794 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 21, + "workflow_seconds": 60, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174594", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 23, + "workflow_seconds": 67, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 102947794, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 102947794, + "unmeasured_tags": [], + "total_bytes": 102947794, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 102947794, + "compressed_size": 102947794, + "size": 102947794 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174595", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-next-turbo/report.md b/results/n8n/cadence-rolling-next-turbo/report.md new file mode 100644 index 000000000..cf464c1d3 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-next-turbo + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 23 | 23–23 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 28 | 28–28 | 102947794 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 2 | 21 | 23 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 5 | 23 | 28 | 102947794 | boringcache-check | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..0386ced48 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 21, + "workflow_seconds": 60, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174594", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] +} diff --git a/results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json new file mode 100644 index 000000000..a2dc5c05d --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 23, + "workflow_seconds": 67, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 102947794, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 102947794, + "unmeasured_tags": [], + "total_bytes": 102947794, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 102947794, + "compressed_size": 102947794, + "size": 102947794 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174595", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] +} diff --git a/results/n8n/cadence-rolling-next-turbo/series.json b/results/n8n/cadence-rolling-next-turbo/series.json new file mode 100644 index 000000000..ff065fef0 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-next-turbo", + "created_at": "2026-10-05T14:29:53Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "b749a682eb42d6c7a423da19c572b9ccd92b020e2c0b11b072e4da3d6a1884c1" +} diff --git a/results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json b/results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json new file mode 100644 index 000000000..84b47892e --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-distroless", + "sample": 1, + "run_id": "37321753318", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5dba4af82ea3265cfe8253b205a2ab4ede49bb19b33981edd5405fe8cd44d094" +} diff --git a/results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..ce1189de7 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless", + "series_id": "cadence-rolling-seed-distroless", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:22Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321753318, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318" +} diff --git a/results/n8n/cadence-rolling-seed-distroless/report.json b/results/n8n/cadence-rolling-seed-distroless/report.json new file mode 100644 index 000000000..45a155dfc --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "d1f18342eadd50145421ea9ed42dd7383e8f463fae9a7416ef1d5661c4c3c73b", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-distroless", + "sample": 1, + "run_id": "37321753318", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5dba4af82ea3265cfe8253b205a2ab4ede49bb19b33981edd5405fe8cd44d094" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 142, + "min": 142, + "max": 142 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 85, + "min": 85, + "max": 85 + }, + "storage": { + "median": 581058531, + "min": 581058531, + "max": 581058531 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 136, + "workflow_seconds": 344, + "total_seconds": 142, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 142 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174357", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 283, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581058531, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581058531, + "unmeasured_tags": [], + "total_bytes": 581058531, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b2e72ac8-7460-474a-bb9e-ed933014c045", + "manifest_root_digest": "sha256:6969aef351b47984ed9c2d79840309b8afbe28b4fcad348e3292ed70fec47403", + "kv_total_size": 0, + "compressed_size": 581058531, + "size": 581058531 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174358", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-distroless/report.md b/results/n8n/cadence-rolling-seed-distroless/report.md new file mode 100644 index 000000000..dc463bf77 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-distroless + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 142 | 142–142 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 85 | 85–85 | 581058531 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 6 | 136 | 142 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 85 | 85 | 581058531 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..ebd380c68 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 136, + "workflow_seconds": 344, + "total_seconds": 142, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 142 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174357", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] +} diff --git a/results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json new file mode 100644 index 000000000..85b486eb9 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 283, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581058531, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581058531, + "unmeasured_tags": [], + "total_bytes": 581058531, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b2e72ac8-7460-474a-bb9e-ed933014c045", + "manifest_root_digest": "sha256:6969aef351b47984ed9c2d79840309b8afbe28b4fcad348e3292ed70fec47403", + "kv_total_size": 0, + "compressed_size": 581058531, + "size": 581058531 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174358", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] +} diff --git a/results/n8n/cadence-rolling-seed-distroless/series.json b/results/n8n/cadence-rolling-seed-distroless/series.json new file mode 100644 index 000000000..975d204ec --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-distroless", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "distroless", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "d1f18342eadd50145421ea9ed42dd7383e8f463fae9a7416ef1d5661c4c3c73b" +} diff --git a/results/n8n/cadence-rolling-seed-docker/completions/37321758035.json b/results/n8n/cadence-rolling-seed-docker/completions/37321758035.json new file mode 100644 index 000000000..0e48a1c95 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/completions/37321758035.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-docker", + "sample": 1, + "run_id": "37321758035", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a282b7867dd2a2cd6c278574761db7beba802f44b877dbacbba2a0058a8010bf" +} diff --git a/results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0a3662357 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker", + "series_id": "cadence-rolling-seed-docker", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:24Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321758035, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035" +} diff --git a/results/n8n/cadence-rolling-seed-docker/report.json b/results/n8n/cadence-rolling-seed-docker/report.json new file mode 100644 index 000000000..0a92caec3 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "5cf26022a9d254a005a5fa1b728266bc712b1b1735d8358d13f3b62629404d3f", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-docker", + "sample": 1, + "run_id": "37321758035", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a282b7867dd2a2cd6c278574761db7beba802f44b877dbacbba2a0058a8010bf" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 361, + "min": 361, + "max": 361 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 233, + "min": 233, + "max": 233 + }, + "storage": { + "median": 483504441, + "min": 483504441, + "max": 483504441 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 352, + "workflow_seconds": 553, + "total_seconds": 361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 361 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174360", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 232, + "workflow_seconds": 399, + "total_seconds": 233, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 233 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483504441, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483504441, + "unmeasured_tags": [], + "total_bytes": 483504441, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "52702ba9-0b6b-4974-be8a-8ac5b7dd91dd", + "manifest_root_digest": "sha256:bc0cec58d12408d3a0ea3da54ac0dfb309704a63e3da68c8c89acf792299c800", + "kv_total_size": 0, + "compressed_size": 483504441, + "size": 483504441 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174359", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-docker/report.md b/results/n8n/cadence-rolling-seed-docker/report.md new file mode 100644 index 000000000..bb3a8ba40 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-docker + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 361 | 361–361 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 233 | 233–233 | 483504441 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 9 | 352 | 361 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 1 | 232 | 233 | 483504441 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..1da972959 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 352, + "workflow_seconds": 553, + "total_seconds": 361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 361 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174360", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] +} diff --git a/results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6c0b63bda --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 232, + "workflow_seconds": 399, + "total_seconds": 233, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 233 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483504441, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483504441, + "unmeasured_tags": [], + "total_bytes": 483504441, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "52702ba9-0b6b-4974-be8a-8ac5b7dd91dd", + "manifest_root_digest": "sha256:bc0cec58d12408d3a0ea3da54ac0dfb309704a63e3da68c8c89acf792299c800", + "kv_total_size": 0, + "compressed_size": 483504441, + "size": 483504441 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174359", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] +} diff --git a/results/n8n/cadence-rolling-seed-docker/series.json b/results/n8n/cadence-rolling-seed-docker/series.json new file mode 100644 index 000000000..df8231c07 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-docker", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "docker", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "5cf26022a9d254a005a5fa1b728266bc712b1b1735d8358d13f3b62629404d3f" +} diff --git a/results/n8n/cadence-rolling-seed-runners/completions/37321762921.json b/results/n8n/cadence-rolling-seed-runners/completions/37321762921.json new file mode 100644 index 000000000..5728911b1 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/completions/37321762921.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-runners", + "sample": 1, + "run_id": "37321762921", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8f8db69bea3b4b1a7cc45f393989c159db64f5e95d90152f7beb9484b6382961" +} diff --git a/results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..c0c90d6e4 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners", + "series_id": "cadence-rolling-seed-runners", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:26Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321762921, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921" +} diff --git a/results/n8n/cadence-rolling-seed-runners/report.json b/results/n8n/cadence-rolling-seed-runners/report.json new file mode 100644 index 000000000..0ceaff2a9 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "1755d2d8b22261156f9b44be5b049be4d4517dfdede31be0c5662ab37afe6b4f", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-runners", + "sample": 1, + "run_id": "37321762921", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8f8db69bea3b4b1a7cc45f393989c159db64f5e95d90152f7beb9484b6382961" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 97, + "min": 97, + "max": 97 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 60, + "min": 60, + "max": 60 + }, + "storage": { + "median": 413880460, + "min": 413880460, + "max": 413880460 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 92, + "workflow_seconds": 297, + "total_seconds": 97, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 97 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174362", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 60, + "workflow_seconds": 217, + "total_seconds": 60, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 60 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413880460, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413880460, + "unmeasured_tags": [], + "total_bytes": 413880460, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eaa46e3-222f-44aa-9cae-ea0496dc904c", + "manifest_root_digest": "sha256:8e9622acb5f3e6a361fc412300476b4c657ff9351ef1e3a458b78995770c188f", + "kv_total_size": 0, + "compressed_size": 413880460, + "size": 413880460 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174361", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-runners/report.md b/results/n8n/cadence-rolling-seed-runners/report.md new file mode 100644 index 000000000..df8fd6d39 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-runners + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 97 | 97–97 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 60 | 60–60 | 413880460 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 92 | 97 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 60 | 60 | 413880460 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..87163af3c --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 92, + "workflow_seconds": 297, + "total_seconds": 97, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 97 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174362", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] +} diff --git a/results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json new file mode 100644 index 000000000..45a4c65eb --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 60, + "workflow_seconds": 217, + "total_seconds": 60, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 60 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413880460, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413880460, + "unmeasured_tags": [], + "total_bytes": 413880460, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eaa46e3-222f-44aa-9cae-ea0496dc904c", + "manifest_root_digest": "sha256:8e9622acb5f3e6a361fc412300476b4c657ff9351ef1e3a458b78995770c188f", + "kv_total_size": 0, + "compressed_size": 413880460, + "size": 413880460 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174361", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] +} diff --git a/results/n8n/cadence-rolling-seed-runners/series.json b/results/n8n/cadence-rolling-seed-runners/series.json new file mode 100644 index 000000000..966a8211d --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-runners", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "runners", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "1755d2d8b22261156f9b44be5b049be4d4517dfdede31be0c5662ab37afe6b4f" +} diff --git a/results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json b/results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json new file mode 100644 index 000000000..c8bb26ea0 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-turbo", + "sample": 1, + "run_id": "37321767409", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c5788a3d03b981f520010fe3cc13c99e5d16a39606f0ff7b0d9437bbb316174c" +} diff --git a/results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..29484dbf8 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo", + "series_id": "cadence-rolling-seed-turbo", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:28Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321767409, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409" +} diff --git a/results/n8n/cadence-rolling-seed-turbo/report.json b/results/n8n/cadence-rolling-seed-turbo/report.json new file mode 100644 index 000000000..7e9a1c4c7 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/report.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "plan_sha256": "844a1fe6776711f983b947113353f8af9d82dc08f2f07180291060668cdfc0fb", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-turbo", + "sample": 1, + "run_id": "37321767409", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c5788a3d03b981f520010fe3cc13c99e5d16a39606f0ff7b0d9437bbb316174c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 85, + "min": 85, + "max": 85 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 87, + "min": 87, + "max": 87 + }, + "storage": { + "median": 46411206, + "min": 46411206, + "max": 46411206 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 125, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174364", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 3, + "build_seconds": 84, + "workflow_seconds": 127, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46411206, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 46411206, + "unmeasured_tags": [], + "total_bytes": 46411206, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46411206, + "compressed_size": 46411206, + "size": 46411206 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174365", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-turbo/report.md b/results/n8n/cadence-rolling-seed-turbo/report.md new file mode 100644 index 000000000..8cb51cff7 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-turbo + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 85 | 85–85 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 87 | 87–87 | 46411206 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 85 | 85 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 3 | 84 | 87 | 46411206 | boringcache-check | miss | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..5262645fe --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 125, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174364", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] +} diff --git a/results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json new file mode 100644 index 000000000..4b246ba3f --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 3, + "build_seconds": 84, + "workflow_seconds": 127, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46411206, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 46411206, + "unmeasured_tags": [], + "total_bytes": 46411206, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46411206, + "compressed_size": 46411206, + "size": 46411206 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174365", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] +} diff --git a/results/n8n/cadence-rolling-seed-turbo/series.json b/results/n8n/cadence-rolling-seed-turbo/series.json new file mode 100644 index 000000000..545a32472 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-turbo", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "844a1fe6776711f983b947113353f8af9d82dc08f2f07180291060668cdfc0fb" +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json b/results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json new file mode 100644 index 000000000..1872773b6 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-ccache-02", + "sample": 1, + "run_id": "37324219008", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c007e7636ce1312c0b26401c97fbfcf189649f4579cdb231c7476c47bf7612cc" +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..4830c9f91 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache", + "series_id": "cadence-rolling-advance-ccache-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:22:45Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324219008, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008" +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/report.json b/results/obs-studio/cadence-rolling-advance-ccache-02/report.json new file mode 100644 index 000000000..9e1babbb9 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/report.json @@ -0,0 +1,490 @@ +{ + "schema_version": 1, + "plan_sha256": "8587641676089f6569ab4c5ef0cd2bd84894ac5368e74ac41018d41a1be9541b", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-ccache-02", + "sample": 1, + "run_id": "37324219008", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c007e7636ce1312c0b26401c97fbfcf189649f4579cdb231c7476c47bf7612cc" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 39.52239645200001, + "min": 39.52239645200001, + "max": 39.52239645200001 + }, + "storage": { + "median": 164323972, + "min": 164323972, + "max": 164323972 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 57.526410286000015, + "min": 57.526410286000015, + "max": 57.526410286000015 + }, + "storage": { + "median": 176275703, + "min": 176275703, + "max": 176275703 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 37.52239645200001, + "total_seconds": 39.52239645200001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 39.52239645200001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 164323972, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "complete": true, + "measured_bytes": 164323972, + "total_bytes": 164323972, + "observations": [ + { + "id": 8523145597, + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "a6730e8f7256fcb7bacf34155bb5305481c02b0e3925c471e4929ecf01b84d07", + "created_at": "2026-10-05T14:30:17.029024Z", + "last_accessed_at": "2026-10-05T14:31:09.615918Z", + "size_in_bytes": 164323972 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174597", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 49.526410286000015, + "total_seconds": 57.526410286000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 57.526410286000015 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 176275703, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 176275703, + "unmeasured_tags": [], + "total_bytes": 176275703, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 176275703, + "compressed_size": 176275703, + "size": 176275703 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174598", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/report.md b/results/obs-studio/cadence-rolling-advance-ccache-02/report.md new file mode 100644 index 000000000..74db081f8 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-advance-ccache-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 39.52239645200001 | 39.52239645200001–39.52239645200001 | 164323972 | 1 | +| Changed-source build | BoringCache | 1 | 57.526410286000015 | 57.526410286000015–57.526410286000015 | 176275703 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 2 | 37.52239645200001 | 39.52239645200001 | 164323972 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 8 | 49.526410286000015 | 57.526410286000015 | 176275703 | boringcache-check | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..70335a6c2 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 37.52239645200001, + "total_seconds": 39.52239645200001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 39.52239645200001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 164323972, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "complete": true, + "measured_bytes": 164323972, + "total_bytes": 164323972, + "observations": [ + { + "id": 8523145597, + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "a6730e8f7256fcb7bacf34155bb5305481c02b0e3925c471e4929ecf01b84d07", + "created_at": "2026-10-05T14:30:17.029024Z", + "last_accessed_at": "2026-10-05T14:31:09.615918Z", + "size_in_bytes": 164323972 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174597", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..86f069ffe --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 49.526410286000015, + "total_seconds": 57.526410286000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 57.526410286000015 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 176275703, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 176275703, + "unmeasured_tags": [], + "total_bytes": 176275703, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 176275703, + "compressed_size": 176275703, + "size": 176275703 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174598", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/series.json b/results/obs-studio/cadence-rolling-advance-ccache-02/series.json new file mode 100644 index 000000000..e08b3b383 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-ccache-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "variant": "ccache", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "8587641676089f6569ab4c5ef0cd2bd84894ac5368e74ac41018d41a1be9541b" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json b/results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json new file mode 100644 index 000000000..b4f677c8d --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode-02", + "sample": 1, + "run_id": "37326340221", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221", + "source_sha": "eaa2b54bf4d09870c8f139850bd2930a28b51f84", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3697df862b900d7681b5db50c705e57271393a0d0b8a050ec75b69e7644f9754" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..a64899f71 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode", + "series_id": "cadence-rolling-advance-xcode-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:38:30Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37326340221, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/report.json b/results/obs-studio/cadence-rolling-advance-xcode-02/report.json new file mode 100644 index 000000000..ad2db42d5 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/report.json @@ -0,0 +1,490 @@ +{ + "schema_version": 1, + "plan_sha256": "71a4a39bcc095f6bcf55fbb038323631597c2974ccb8b14c437ed5ee04d4c9d8", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode-02", + "sample": 1, + "run_id": "37326340221", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221", + "source_sha": "eaa2b54bf4d09870c8f139850bd2930a28b51f84", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3697df862b900d7681b5db50c705e57271393a0d0b8a050ec75b69e7644f9754" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 189.99793799999998, + "min": 189.99793799999998, + "max": 189.99793799999998 + }, + "storage": { + "median": 309445116, + "min": 309445116, + "max": 309445116 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 169.57710600000001, + "min": 169.57710600000001, + "max": 169.57710600000001 + }, + "storage": { + "median": 1004724519, + "min": 1004724519, + "max": 1004724519 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 178.99793799999998, + "total_seconds": 189.99793799999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 189.99793799999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 309445116, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "complete": true, + "measured_bytes": 309445116, + "total_bytes": 309445116, + "observations": [ + { + "id": 8524219730, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:51:56.940267Z", + "last_accessed_at": "2026-10-05T14:53:14.694488Z", + "size_in_bytes": 309445116 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174657", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 23, + "build_seconds": 146.57710600000001, + "total_seconds": 169.57710600000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 169.57710600000001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1004724519, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 1004724519, + "unmeasured_tags": [], + "total_bytes": 1004724519, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 1004724519, + "compressed_size": 1004724519, + "size": 1004724519 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174656", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/report.md b/results/obs-studio/cadence-rolling-advance-xcode-02/report.md new file mode 100644 index 000000000..82ae5f62f --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-advance-xcode-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 189.99793799999998 | 189.99793799999998–189.99793799999998 | 309445116 | 1 | +| Changed-source build | BoringCache | 1 | 169.57710600000001 | 169.57710600000001–169.57710600000001 | 1004724519 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 11 | 178.99793799999998 | 189.99793799999998 | 309445116 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 23 | 146.57710600000001 | 169.57710600000001 | 1004724519 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..53e9d8a4f --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 178.99793799999998, + "total_seconds": 189.99793799999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 189.99793799999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 309445116, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "complete": true, + "measured_bytes": 309445116, + "total_bytes": 309445116, + "observations": [ + { + "id": 8524219730, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:51:56.940267Z", + "last_accessed_at": "2026-10-05T14:53:14.694488Z", + "size_in_bytes": 309445116 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174657", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f846478da --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 23, + "build_seconds": 146.57710600000001, + "total_seconds": 169.57710600000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 169.57710600000001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1004724519, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 1004724519, + "unmeasured_tags": [], + "total_bytes": 1004724519, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 1004724519, + "compressed_size": 1004724519, + "size": 1004724519 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174656", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/series.json b/results/obs-studio/cadence-rolling-advance-xcode-02/series.json new file mode 100644 index 000000000..b9c56b27a --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode-02", + "created_at": "2026-10-05T14:38:27Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "71a4a39bcc095f6bcf55fbb038323631597c2974ccb8b14c437ed5ee04d4c9d8" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json b/results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json new file mode 100644 index 000000000..280adaeca --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "sample": 1, + "run_id": "37323234591", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "affe5bec73070abd855bf7db61b49d6eeb974d240029ffa55c02074c2486b458" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..c2a379091 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode", + "series_id": "cadence-rolling-advance-xcode", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:15:23Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323234591, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/report.json b/results/obs-studio/cadence-rolling-advance-xcode/report.json new file mode 100644 index 000000000..0d4426c03 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/report.json @@ -0,0 +1,278 @@ +{ + "schema_version": 1, + "plan_sha256": "c42ea925dc2d15cb5a24ef14e27b267001b6bf8b9ed29989e51c7e927e3875e4", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "sample": 1, + "run_id": "37323234591", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "affe5bec73070abd855bf7db61b49d6eeb974d240029ffa55c02074c2486b458" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "sample": 1, + "run_id": "37323234591", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "affe5bec73070abd855bf7db61b49d6eeb974d240029ffa55c02074c2486b458" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 129.465303, + "min": 129.465303, + "max": 129.465303 + }, + "storage": { + "median": 310246072, + "min": 310246072, + "max": 310246072 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "215aea66ef6c9adae46be8c8f6d89395e0ffdffcb64b7184ef9f0157512506cc", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 124.465303, + "total_seconds": 129.465303, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 129.465303 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 310246072, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "complete": true, + "measured_bytes": 310246072, + "total_bytes": 310246072, + "observations": [ + { + "id": 8522777408, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:22:57.114691Z", + "last_accessed_at": "2026-10-05T14:26:07.297970Z", + "size_in_bytes": 310246072 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323234591", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174567", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37323234591-1", + "series": { + "id": "cadence-rolling-advance-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323234591" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/report.md b/results/obs-studio/cadence-rolling-advance-xcode/report.md new file mode 100644 index 000000000..272a5f564 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/report.md @@ -0,0 +1,37 @@ +# obs-studio: cadence-rolling-advance-xcode + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 129.465303 | 129.465303–129.465303 | 310246072 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +Run 37323234591 failed completion checks: + +- Workflow concluded failure +- BoringCache obs-studio xcode commit: failure +- obs-studio rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 124.465303 | 129.465303 | 310246072 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..1e06e516e --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "215aea66ef6c9adae46be8c8f6d89395e0ffdffcb64b7184ef9f0157512506cc", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 124.465303, + "total_seconds": 129.465303, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 129.465303 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 310246072, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "complete": true, + "measured_bytes": 310246072, + "total_bytes": 310246072, + "observations": [ + { + "id": 8522777408, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:22:57.114691Z", + "last_accessed_at": "2026-10-05T14:26:07.297970Z", + "size_in_bytes": 310246072 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323234591", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174567", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37323234591-1", + "series": { + "id": "cadence-rolling-advance-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323234591" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/series.json b/results/obs-studio/cadence-rolling-advance-xcode/series.json new file mode 100644 index 000000000..62f430792 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "215aea66ef6c9adae46be8c8f6d89395e0ffdffcb64b7184ef9f0157512506cc", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "c42ea925dc2d15cb5a24ef14e27b267001b6bf8b9ed29989e51c7e927e3875e4" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json b/results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json new file mode 100644 index 000000000..6c37ccd69 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache-02", + "sample": 1, + "run_id": "37323465858", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858", + "source_sha": "c19b136d6863c141212af5aa5620750e485f3009", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "aa2fa4b21b48ff7c23352fecee8574cde35af7f040add1063089e2906d2fe45e" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..0df92f38c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache", + "series_id": "cadence-rolling-seed-ccache-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:17:07Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323465858, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/report.json b/results/obs-studio/cadence-rolling-seed-ccache-02/report.json new file mode 100644 index 000000000..e7cc2fcc8 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/report.json @@ -0,0 +1,470 @@ +{ + "schema_version": 1, + "plan_sha256": "5340d6e1868ad898bd6953d3231a6971cf4ea0fb838c12181a663edbdd6191b2", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache-02", + "sample": 1, + "run_id": "37323465858", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858", + "source_sha": "c19b136d6863c141212af5aa5620750e485f3009", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "aa2fa4b21b48ff7c23352fecee8574cde35af7f040add1063089e2906d2fe45e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 644.054199763, + "min": 644.054199763, + "max": 644.054199763 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 631.7126567849999, + "min": 631.7126567849999, + "max": 631.7126567849999 + }, + "storage": { + "median": 172201371, + "min": 172201371, + "max": 172201371 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 643.054199763, + "total_seconds": 644.054199763, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 644.054199763 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174504", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 629.7126567849999, + "total_seconds": 631.7126567849999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 631.7126567849999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 172201371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 172201371, + "unmeasured_tags": [], + "total_bytes": 172201371, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 172201371, + "compressed_size": 172201371, + "size": 172201371 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174505", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/report.md b/results/obs-studio/cadence-rolling-seed-ccache-02/report.md new file mode 100644 index 000000000..0548dbb09 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-seed-ccache-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 644.054199763 | 644.054199763–644.054199763 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 631.7126567849999 | 631.7126567849999–631.7126567849999 | 172201371 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 1 | 643.054199763 | 644.054199763 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 2 | 629.7126567849999 | 631.7126567849999 | 172201371 | boringcache-check | miss | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..f4287cda3 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 643.054199763, + "total_seconds": 644.054199763, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 644.054199763 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174504", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..9c89fe397 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 629.7126567849999, + "total_seconds": 631.7126567849999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 631.7126567849999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 172201371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 172201371, + "unmeasured_tags": [], + "total_bytes": 172201371, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 172201371, + "compressed_size": 172201371, + "size": 172201371 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174505", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/series.json b/results/obs-studio/cadence-rolling-seed-ccache-02/series.json new file mode 100644 index 000000000..c70a838a7 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache-02", + "created_at": "2026-10-05T14:17:04Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "variant": "ccache", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "5340d6e1868ad898bd6953d3231a6971cf4ea0fb838c12181a663edbdd6191b2" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json b/results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json new file mode 100644 index 000000000..bbc958a7c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "sample": 1, + "run_id": "37322616359", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio ccache commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "5320b399ed49d4a3e415fe897b8181b80e68beb26a4ffe76ba463c02f808421c" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..35f32d85c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache", + "cache_tool": "ccache", + "series_id": "cadence-rolling-seed-ccache", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:10:46Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322616359, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/report.json b/results/obs-studio/cadence-rolling-seed-ccache/report.json new file mode 100644 index 000000000..79073cf9e --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/report.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "plan_sha256": "bd29f1e4edeb3be507ce538fa07532051b53d8d111665c7717bc6121e00d8657", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "sample": 1, + "run_id": "37322616359", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio ccache commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "5320b399ed49d4a3e415fe897b8181b80e68beb26a4ffe76ba463c02f808421c" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "sample": 1, + "run_id": "37322616359", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio ccache commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "5320b399ed49d4a3e415fe897b8181b80e68beb26a4ffe76ba463c02f808421c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 577.6759015939999, + "min": 577.6759015939999, + "max": 577.6759015939999 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 577.6759015939999, + "total_seconds": 577.6759015939999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 577.6759015939999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322616359", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174437", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37322616359-1", + "series": { + "id": "cadence-rolling-seed-ccache", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322616359" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/report.md b/results/obs-studio/cadence-rolling-seed-ccache/report.md new file mode 100644 index 000000000..efa10be9f --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/report.md @@ -0,0 +1,37 @@ +# obs-studio: cadence-rolling-seed-ccache + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 577.6759015939999 | 577.6759015939999–577.6759015939999 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +Run 37322616359 failed completion checks: + +- Workflow concluded failure +- BoringCache obs-studio ccache commit: failure +- obs-studio rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 577.6759015939999 | 577.6759015939999 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..5be1b4f74 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 577.6759015939999, + "total_seconds": 577.6759015939999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 577.6759015939999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322616359", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174437", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37322616359-1", + "series": { + "id": "cadence-rolling-seed-ccache", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322616359" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/series.json b/results/obs-studio/cadence-rolling-seed-ccache/series.json new file mode 100644 index 000000000..cb4e0e04a --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "variant": "ccache", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache", + "cache_tool": "ccache" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "bd29f1e4edeb3be507ce538fa07532051b53d8d111665c7717bc6121e00d8657" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json b/results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json new file mode 100644 index 000000000..2e44dcfc5 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode-02", + "sample": 1, + "run_id": "37326133177", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177", + "source_sha": "e6e7a3da41e0e94deaae9dbcb4ed530aaa3c59f1", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1494eea0f15a03a25bb4547ac541b315520ae761dc4a1cd57d90d9a731899297" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..6e06f0b7a --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode", + "series_id": "cadence-rolling-seed-xcode-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:36:58Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37326133177, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/report.json b/results/obs-studio/cadence-rolling-seed-xcode-02/report.json new file mode 100644 index 000000000..f34ae8ca8 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/report.json @@ -0,0 +1,470 @@ +{ + "schema_version": 1, + "plan_sha256": "37406699c5188bce11315d38f171cad90fec8af90d48603b103699b9dc475c84", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode-02", + "sample": 1, + "run_id": "37326133177", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177", + "source_sha": "e6e7a3da41e0e94deaae9dbcb4ed530aaa3c59f1", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1494eea0f15a03a25bb4547ac541b315520ae761dc4a1cd57d90d9a731899297" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 710.543963, + "min": 710.543963, + "max": 710.543963 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 528.281434, + "min": 528.281434, + "max": 528.281434 + }, + "storage": { + "median": 983997286, + "min": 983997286, + "max": 983997286 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 710.543963, + "total_seconds": 710.543963, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 710.543963 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174629", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 523.281434, + "total_seconds": 528.281434, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 528.281434 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 983997286, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 983997286, + "unmeasured_tags": [], + "total_bytes": 983997286, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 983997286, + "compressed_size": 983997286, + "size": 983997286 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174630", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/report.md b/results/obs-studio/cadence-rolling-seed-xcode-02/report.md new file mode 100644 index 000000000..5ef19aa96 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-seed-xcode-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 710.543963 | 710.543963–710.543963 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 528.281434 | 528.281434–528.281434 | 983997286 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 710.543963 | 710.543963 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 5 | 523.281434 | 528.281434 | 983997286 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..8009e6252 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 710.543963, + "total_seconds": 710.543963, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 710.543963 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174629", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..3336d886c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 523.281434, + "total_seconds": 528.281434, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 528.281434 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 983997286, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 983997286, + "unmeasured_tags": [], + "total_bytes": 983997286, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 983997286, + "compressed_size": 983997286, + "size": 983997286 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174630", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/series.json b/results/obs-studio/cadence-rolling-seed-xcode-02/series.json new file mode 100644 index 000000000..b97f5844b --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode-02", + "created_at": "2026-10-05T14:36:56Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "37406699c5188bce11315d38f171cad90fec8af90d48603b103699b9dc475c84" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json b/results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json new file mode 100644 index 000000000..4562bc2f6 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "sample": 1, + "run_id": "37322621030", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "167a4879396a537d636434a85dc880174b79075432c1f84b3bc26955594fbd02" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..0fd58a6a0 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode", + "series_id": "cadence-rolling-seed-xcode", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:10:49Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322621030, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/report.json b/results/obs-studio/cadence-rolling-seed-xcode/report.json new file mode 100644 index 000000000..7a262f88a --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/report.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "plan_sha256": "152102f6ca51dcfd61c214b07455c9a2fccc8aba13dc4ce089c3a6b016ca52a9", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "sample": 1, + "run_id": "37322621030", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "167a4879396a537d636434a85dc880174b79075432c1f84b3bc26955594fbd02" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "sample": 1, + "run_id": "37322621030", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "167a4879396a537d636434a85dc880174b79075432c1f84b3bc26955594fbd02" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 550.518804, + "min": 550.518804, + "max": 550.518804 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 550.518804, + "total_seconds": 550.518804, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 550.518804 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322621030", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174440", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37322621030-1", + "series": { + "id": "cadence-rolling-seed-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322621030" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/report.md b/results/obs-studio/cadence-rolling-seed-xcode/report.md new file mode 100644 index 000000000..532eba981 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/report.md @@ -0,0 +1,37 @@ +# obs-studio: cadence-rolling-seed-xcode + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 550.518804 | 550.518804–550.518804 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +Run 37322621030 failed completion checks: + +- Workflow concluded failure +- BoringCache obs-studio xcode commit: failure +- obs-studio rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 550.518804 | 550.518804 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..eb36c893f --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 550.518804, + "total_seconds": 550.518804, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 550.518804 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322621030", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174440", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37322621030-1", + "series": { + "id": "cadence-rolling-seed-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322621030" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/series.json b/results/obs-studio/cadence-rolling-seed-xcode/series.json new file mode 100644 index 000000000..87f6e29c9 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "152102f6ca51dcfd61c214b07455c9a2fccc8aba13dc4ce089c3a6b016ca52a9" +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json b/results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json new file mode 100644 index 000000000..c85407c6f --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324223748", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "4834dc48812d1ab6f8e6114630757a20ed7661c1b4d1493705efe89e8b3d00b6" +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..af63d983c --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:47Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324223748, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748" +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/report.json b/results/opencut-moon/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..d84d7dc86 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/report.json @@ -0,0 +1,220 @@ +{ + "schema_version": 1, + "plan_sha256": "78b7796942758265e9ac7bab7cce6a7a579307fdafc10566522875719f33976d", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324223748", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "4834dc48812d1ab6f8e6114630757a20ed7661c1b4d1493705efe89e8b3d00b6" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324223748", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "4834dc48812d1ab6f8e6114630757a20ed7661c1b4d1493705efe89e8b3d00b6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10101780600000154, + "build_seconds": 9.075982147000005, + "workflow_seconds": null, + "total_seconds": 9.176999953000006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0015317619999990484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.176999953000006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324223748", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174547", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324223748-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324223748" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/report.md b/results/opencut-moon/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..7380ccac5 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/report.md @@ -0,0 +1,38 @@ +# opencut-moon: cadence-rolling-advance-02 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37324223748 failed completion checks: + +- Workflow concluded failure +- BoringCache opencut-moon commit: failure +- opencut-moon rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10101780600000154 | 9.075982147000005 | 9.176999953000006 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..147650935 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10101780600000154, + "build_seconds": 9.075982147000005, + "workflow_seconds": null, + "total_seconds": 9.176999953000006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0015317619999990484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.176999953000006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324223748", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174547", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324223748-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324223748" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/series.json b/results/opencut-moon/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..b091032e7 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "78b7796942758265e9ac7bab7cce6a7a579307fdafc10566522875719f33976d" +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json b/results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json new file mode 100644 index 000000000..5de34bffb --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325287254", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "b444b24fbab3a1d42d2449737aa46e619bf28e4a2dfc85f3daab79449e796e0c" +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..cd66b61d2 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3", + "series_id": "cadence-rolling-advance-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:30:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325287254, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254" +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/report.json b/results/opencut-moon/cadence-rolling-advance-03/report.json new file mode 100644 index 000000000..fc6b8ff13 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "583701a786de16d224f09043f85430491ec6c9f58fdd9b4431b984561cc22f3a", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325287254", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "b444b24fbab3a1d42d2449737aa46e619bf28e4a2dfc85f3daab79449e796e0c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108759999999961, + "build_seconds": 13.598188859999993, + "workflow_seconds": null, + "total_seconds": 13.699276459999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0017177939999868386, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.699276459999993 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174602", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20152445499999772, + "build_seconds": 14.694388483000012, + "workflow_seconds": null, + "total_seconds": 14.89591293800001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.35811930500000244, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.89591293800001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174603", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/report.md b/results/opencut-moon/cadence-rolling-advance-03/report.md new file mode 100644 index 000000000..50e748f52 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-advance-03 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10108759999999961 | 13.598188859999993 | 13.699276459999993 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20152445499999772 | 14.694388483000012 | 14.89591293800001 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..62c62dbdc --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108759999999961, + "build_seconds": 13.598188859999993, + "workflow_seconds": null, + "total_seconds": 13.699276459999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0017177939999868386, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.699276459999993 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174602", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..7b0554019 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20152445499999772, + "build_seconds": 14.694388483000012, + "workflow_seconds": null, + "total_seconds": 14.89591293800001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.35811930500000244, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.89591293800001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174603", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/series.json b/results/opencut-moon/cadence-rolling-advance-03/series.json new file mode 100644 index 000000000..bfc2814ff --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-03", + "created_at": "2026-10-05T14:30:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "583701a786de16d224f09043f85430491ec6c9f58fdd9b4431b984561cc22f3a" +} diff --git a/results/opencut-moon/cadence-rolling-advance/completions/37323159608.json b/results/opencut-moon/cadence-rolling-advance/completions/37323159608.json new file mode 100644 index 000000000..a3adbdc91 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/completions/37323159608.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323159608", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "86eaebf0d054dc81b425ce0ae87c8bbce0cc185b4bad17ea13dc7b2e1ef1fd20" +} diff --git a/results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..f37153388 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:52Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323159608, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608" +} diff --git a/results/opencut-moon/cadence-rolling-advance/report.json b/results/opencut-moon/cadence-rolling-advance/report.json new file mode 100644 index 000000000..bb2a2ceeb --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "b0cbc49ae9e5b938a1a5379204ddc56b2d1349af8b6ff6452f4ab4790776ebfe", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323159608", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "86eaebf0d054dc81b425ce0ae87c8bbce0cc185b4bad17ea13dc7b2e1ef1fd20" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323159608", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "86eaebf0d054dc81b425ce0ae87c8bbce0cc185b4bad17ea13dc7b2e1ef1fd20" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7c8bfdbc420b4d1e9e9edfd49359f619498d3783a60d1e01da270448115f7553", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.3016585379999981, + "build_seconds": 11.28177397799999, + "workflow_seconds": null, + "total_seconds": 11.583432515999988, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4176102280000009, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 11.583432515999988 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323159608", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174486", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323159608-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323159608" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-advance/report.md b/results/opencut-moon/cadence-rolling-advance/report.md new file mode 100644 index 000000000..bb9232332 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# opencut-moon: cadence-rolling-advance + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323159608 failed completion checks: + +- Workflow concluded failure +- bazel-remote opencut-moon commit: failure +- opencut-moon rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.3016585379999981 | 11.28177397799999 | 11.583432515999988 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..506d5cf95 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7c8bfdbc420b4d1e9e9edfd49359f619498d3783a60d1e01da270448115f7553", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.3016585379999981, + "build_seconds": 11.28177397799999, + "workflow_seconds": null, + "total_seconds": 11.583432515999988, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4176102280000009, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 11.583432515999988 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323159608", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174486", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323159608-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323159608" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance/series.json b/results/opencut-moon/cadence-rolling-advance/series.json new file mode 100644 index 000000000..efe4bd12e --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "7c8bfdbc420b4d1e9e9edfd49359f619498d3783a60d1e01da270448115f7553", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b0cbc49ae9e5b938a1a5379204ddc56b2d1349af8b6ff6452f4ab4790776ebfe" +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json b/results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json new file mode 100644 index 000000000..0344e8d5f --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324020999", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324020999", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "52e63f033403920d3155d562b3e02d8d1e7c85fb46197895ef027dee1237cd9c" +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..a5091a107 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:17Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324020999, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324020999" +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/report.json b/results/opencut-moon/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..ea13da69d --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "a8ca6ebc122865dc73ab748ade79bf221a2a2c16c5e1fbd86e9e7fb465e4fc9f", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324020999", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324020999", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "52e63f033403920d3155d562b3e02d8d1e7c85fb46197895ef027dee1237cd9c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10247405000001208, + "build_seconds": 15.200883040999997, + "workflow_seconds": null, + "total_seconds": 15.30335709100001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013556339999922784, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.30335709100001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174528", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011381649999521, + "build_seconds": 16.51002464100003, + "workflow_seconds": null, + "total_seconds": 16.61116280599998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.45349295599999095, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.61116280599998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174529", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/report.md b/results/opencut-moon/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..e29c5b0e2 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-seed-02 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10247405000001208 | 15.200883040999997 | 15.30335709100001 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.1011381649999521 | 16.51002464100003 | 16.61116280599998 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..43e3f766b --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10247405000001208, + "build_seconds": 15.200883040999997, + "workflow_seconds": null, + "total_seconds": 15.30335709100001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013556339999922784, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.30335709100001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174528", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..445552c2e --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011381649999521, + "build_seconds": 16.51002464100003, + "workflow_seconds": null, + "total_seconds": 16.61116280599998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.45349295599999095, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.61116280599998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174529", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/series.json b/results/opencut-moon/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..d26d72f17 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "a8ca6ebc122865dc73ab748ade79bf221a2a2c16c5e1fbd86e9e7fb465e4fc9f" +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json b/results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json new file mode 100644 index 000000000..88e47abb0 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325140807", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ab7c1d13b79fedd9c254e3d5d75ef4b45a857dd5f507e1a35d474d6a9faa7b7e" +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..39b3845a7 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3", + "series_id": "cadence-rolling-seed-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:29:36Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325140807, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807" +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/report.json b/results/opencut-moon/cadence-rolling-seed-03/report.json new file mode 100644 index 000000000..acdd81f7c --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "c6e4c83124eebe212e820a4b00cbd5cd89edd24af9e660b4cf54e852544a2d78", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325140807", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ab7c1d13b79fedd9c254e3d5d75ef4b45a857dd5f507e1a35d474d6a9faa7b7e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.101062933999998, + "build_seconds": 14.702613067999998, + "workflow_seconds": null, + "total_seconds": 14.803676001999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016717000000028293, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.803676001999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174593", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014604100000028, + "build_seconds": 14.595511775999995, + "workflow_seconds": null, + "total_seconds": 14.796972185999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.1871384540000065, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.796972185999998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174592", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/report.md b/results/opencut-moon/cadence-rolling-seed-03/report.md new file mode 100644 index 000000000..71ab1926f --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-seed-03 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.101062933999998 | 14.702613067999998 | 14.803676001999996 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.2014604100000028 | 14.595511775999995 | 14.796972185999998 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..bdc8abcd8 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.101062933999998, + "build_seconds": 14.702613067999998, + "workflow_seconds": null, + "total_seconds": 14.803676001999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016717000000028293, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.803676001999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174593", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..85fa0fa29 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014604100000028, + "build_seconds": 14.595511775999995, + "workflow_seconds": null, + "total_seconds": 14.796972185999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.1871384540000065, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.796972185999998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174592", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/series.json b/results/opencut-moon/cadence-rolling-seed-03/series.json new file mode 100644 index 000000000..51de22bf5 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-03", + "created_at": "2026-10-05T14:29:29Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "c6e4c83124eebe212e820a4b00cbd5cd89edd24af9e660b4cf54e852544a2d78" +} diff --git a/results/opencut-moon/cadence-rolling-seed/completions/37322625973.json b/results/opencut-moon/cadence-rolling-seed/completions/37322625973.json new file mode 100644 index 000000000..19b8d39f9 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/completions/37322625973.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322625973", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322625973", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7dd25aa5714d0dfeef7251b47ca48503eed5cc091bc592fa91b24ed7ede23a1c" +} diff --git a/results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..323260711 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:51Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322625973, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322625973" +} diff --git a/results/opencut-moon/cadence-rolling-seed/report.json b/results/opencut-moon/cadence-rolling-seed/report.json new file mode 100644 index 000000000..bb008883d --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "4306a91ea3011928b3bc572cd3c97705532f20966ab26f7a98a287b5c94abb67", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322625973", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322625973", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7dd25aa5714d0dfeef7251b47ca48503eed5cc091bc592fa91b24ed7ede23a1c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009767549999907, + "build_seconds": 15.103258492999984, + "workflow_seconds": null, + "total_seconds": 15.204235247999975, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001731152000019165, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.204235247999975 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174446", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20123272899999733, + "build_seconds": 16.601319771000007, + "workflow_seconds": null, + "total_seconds": 16.802552500000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.310330332999996, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.802552500000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174447", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-seed/report.md b/results/opencut-moon/cadence-rolling-seed/report.md new file mode 100644 index 000000000..ce5a080a8 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-seed + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1009767549999907 | 15.103258492999984 | 15.204235247999975 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20123272899999733 | 16.601319771000007 | 16.802552500000004 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..35a574729 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009767549999907, + "build_seconds": 15.103258492999984, + "workflow_seconds": null, + "total_seconds": 15.204235247999975, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001731152000019165, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.204235247999975 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174446", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d3e87da84 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20123272899999733, + "build_seconds": 16.601319771000007, + "workflow_seconds": null, + "total_seconds": 16.802552500000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.310330332999996, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.802552500000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174447", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed/series.json b/results/opencut-moon/cadence-rolling-seed/series.json new file mode 100644 index 000000000..9fa7665b4 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "4306a91ea3011928b3bc572cd3c97705532f20966ab26f7a98a287b5c94abb67" +} diff --git a/results/posthog/cadence-rolling-advance-layers/completions/37323164071.json b/results/posthog/cadence-rolling-advance-layers/completions/37323164071.json new file mode 100644 index 000000000..24b1ef939 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/completions/37323164071.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-advance-layers", + "sample": 1, + "run_id": "37323164071", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c6579870875c3dc988763f5253e3db3ae58ed14ff09d4f9a7663af89e3baeb54" +} diff --git a/results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json b/results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0224a1f26 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers", + "series_id": "cadence-rolling-advance-layers", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:53Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323164071, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071" +} diff --git a/results/posthog/cadence-rolling-advance-layers/report.json b/results/posthog/cadence-rolling-advance-layers/report.json new file mode 100644 index 000000000..8bca1180b --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/report.json @@ -0,0 +1,509 @@ +{ + "schema_version": 1, + "plan_sha256": "9c9f99841fba0c7a6826bcd1d39d5d5268e5aa213c5e838ed0bf08b4e7d23ecb", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-advance-layers", + "sample": 1, + "run_id": "37323164071", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c6579870875c3dc988763f5253e3db3ae58ed14ff09d4f9a7663af89e3baeb54" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 2068, + "min": 2068, + "max": 2068 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 904, + "min": 904, + "max": 904 + }, + "storage": { + "median": 6262453567, + "min": 6262453567, + "max": 6262453567 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 2060, + "workflow_seconds": 2084, + "total_seconds": 2068, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2068 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174633", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 904, + "workflow_seconds": 936, + "total_seconds": 904, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 904 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6262453567, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6262453567, + "unmeasured_tags": [], + "total_bytes": 6262453567, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "37d3eb0c-7f24-4221-93c8-7cb379c43930", + "manifest_root_digest": "sha256:6fadbfd68c6194cc8d8992153cdb027cdebf4ccba7fa6b02b1bda1429e74977a", + "kv_total_size": 0, + "compressed_size": 6262453567, + "size": 6262453567 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174632", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/cadence-rolling-advance-layers/report.md b/results/posthog/cadence-rolling-advance-layers/report.md new file mode 100644 index 000000000..da070d0c9 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/report.md @@ -0,0 +1,29 @@ +# posthog: cadence-rolling-advance-layers + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 2068 | 2068–2068 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 904 | 904–904 | 6262453567 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 8 | 2060 | 2068 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 904 | 904 | 6262453567 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json b/results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..79146c225 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json @@ -0,0 +1,199 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 2060, + "workflow_seconds": 2084, + "total_seconds": 2068, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2068 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174633", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] +} diff --git a/results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json b/results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json new file mode 100644 index 000000000..1cb68df17 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json @@ -0,0 +1,245 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 904, + "workflow_seconds": 936, + "total_seconds": 904, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 904 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6262453567, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6262453567, + "unmeasured_tags": [], + "total_bytes": 6262453567, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "37d3eb0c-7f24-4221-93c8-7cb379c43930", + "manifest_root_digest": "sha256:6fadbfd68c6194cc8d8992153cdb027cdebf4ccba7fa6b02b1bda1429e74977a", + "kv_total_size": 0, + "compressed_size": 6262453567, + "size": 6262453567 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174632", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] +} diff --git a/results/posthog/cadence-rolling-advance-layers/series.json b/results/posthog/cadence-rolling-advance-layers/series.json new file mode 100644 index 000000000..c34a1df67 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-advance-layers", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "9c9f99841fba0c7a6826bcd1d39d5d5268e5aa213c5e838ed0bf08b4e7d23ecb" +} diff --git a/results/posthog/cadence-rolling-seed-layers/completions/37321772493.json b/results/posthog/cadence-rolling-seed-layers/completions/37321772493.json new file mode 100644 index 000000000..d81853290 --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/completions/37321772493.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-seed-layers", + "sample": 1, + "run_id": "37321772493", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "663ed2039ee1f1f48a2cf710d82909d9f3b7530f78dc1fea94b2a5e984134623" +} diff --git a/results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json b/results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..122333a33 --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers", + "series_id": "cadence-rolling-seed-layers", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:30Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321772493, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493" +} diff --git a/results/posthog/cadence-rolling-seed-layers/report.json b/results/posthog/cadence-rolling-seed-layers/report.json new file mode 100644 index 000000000..7fee23e4b --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/report.json @@ -0,0 +1,509 @@ +{ + "schema_version": 1, + "plan_sha256": "4ffb50f0d4f2613dc4e123d6d90cdbef454111c9a523e945c745d094180bdd7c", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-seed-layers", + "sample": 1, + "run_id": "37321772493", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "663ed2039ee1f1f48a2cf710d82909d9f3b7530f78dc1fea94b2a5e984134623" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 2088, + "min": 2088, + "max": 2088 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 814, + "min": 814, + "max": 814 + }, + "storage": { + "median": 6243631290, + "min": 6243631290, + "max": 6243631290 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 2077, + "workflow_seconds": 2134, + "total_seconds": 2088, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2088 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174366", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 814, + "workflow_seconds": 832, + "total_seconds": 814, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 814 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6243631290, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6243631290, + "unmeasured_tags": [], + "total_bytes": 6243631290, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b9df360b-3d34-4c58-9153-9689fc78eccb", + "manifest_root_digest": "sha256:65d4fc042f3ae490b0a4ab7d39f9d28283149ce7afb17582bcd8618e563e3b41", + "kv_total_size": 0, + "compressed_size": 6243631290, + "size": 6243631290 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174367", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/cadence-rolling-seed-layers/report.md b/results/posthog/cadence-rolling-seed-layers/report.md new file mode 100644 index 000000000..95c24340d --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/report.md @@ -0,0 +1,29 @@ +# posthog: cadence-rolling-seed-layers + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 2088 | 2088–2088 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 814 | 814–814 | 6243631290 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 11 | 2077 | 2088 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 814 | 814 | 6243631290 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json b/results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..475f7d27f --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json @@ -0,0 +1,199 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 2077, + "workflow_seconds": 2134, + "total_seconds": 2088, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2088 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174366", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] +} diff --git a/results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json b/results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f27205b6a --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json @@ -0,0 +1,245 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 814, + "workflow_seconds": 832, + "total_seconds": 814, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 814 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6243631290, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6243631290, + "unmeasured_tags": [], + "total_bytes": 6243631290, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b9df360b-3d34-4c58-9153-9689fc78eccb", + "manifest_root_digest": "sha256:65d4fc042f3ae490b0a4ab7d39f9d28283149ce7afb17582bcd8618e563e3b41", + "kv_total_size": 0, + "compressed_size": 6243631290, + "size": 6243631290 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174367", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] +} diff --git a/results/posthog/cadence-rolling-seed-layers/series.json b/results/posthog/cadence-rolling-seed-layers/series.json new file mode 100644 index 000000000..e7a16d3cf --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-seed-layers", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "4ffb50f0d4f2613dc4e123d6d90cdbef454111c9a523e945c745d094180bdd7c" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json b/results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json new file mode 100644 index 000000000..09948412e --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324228460", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c93fe2621f6909406e9f055215f2c55d6346b2927045b0d52c076c4f404ca55" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..29c47d561 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:50Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324228460, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/report.json b/results/stackstorm-pants/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..8e1f00171 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "ea0b66d0766668904cd63dc76ff37096b632076ea4e27769912bf66353928a78", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324228460", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c93fe2621f6909406e9f055215f2c55d6346b2927045b0d52c076c4f404ca55" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10122090600000888, + "build_seconds": 13.451143174999999, + "workflow_seconds": null, + "total_seconds": 13.552364081000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0018756030000020019, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.552364081000007 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174561", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10115972500000225, + "build_seconds": 13.277457289000012, + "workflow_seconds": null, + "total_seconds": 13.378617014000014, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.2545980109999846, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.378617014000014 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174562", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/report.md b/results/stackstorm-pants/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..92d08b5c5 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/report.md @@ -0,0 +1,30 @@ +# stackstorm-pants: cadence-rolling-advance-02 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10122090600000888 | 13.451143174999999 | 13.552364081000007 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.10115972500000225 | 13.277457289000012 | 13.378617014000014 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..9465eddab --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10122090600000888, + "build_seconds": 13.451143174999999, + "workflow_seconds": null, + "total_seconds": 13.552364081000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0018756030000020019, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.552364081000007 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174561", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..7f3ae1445 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10115972500000225, + "build_seconds": 13.277457289000012, + "workflow_seconds": null, + "total_seconds": 13.378617014000014, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.2545980109999846, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.378617014000014 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174562", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/series.json b/results/stackstorm-pants/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..ac5f85e39 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "ea0b66d0766668904cd63dc76ff37096b632076ea4e27769912bf66353928a78" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json b/results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json new file mode 100644 index 000000000..d4e0605c6 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323169828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote stackstorm-pants commit: failure", + "stackstorm-pants rolling report: failure" + ], + "evidence_manifest_sha256": "ec94c08b8d5b64fa278e36cb795b9fac3c01f887411aeada029a317606e965de" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..d901a92c4 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:55Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323169828, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/report.json b/results/stackstorm-pants/cadence-rolling-advance/report.json new file mode 100644 index 000000000..08611b3e9 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "cbd4ab69f17a3ca86a1858a9378069d9fe3b8a14bdfe725bc465c02b03b44f5f", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323169828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote stackstorm-pants commit: failure", + "stackstorm-pants rolling report: failure" + ], + "evidence_manifest_sha256": "ec94c08b8d5b64fa278e36cb795b9fac3c01f887411aeada029a317606e965de" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323169828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote stackstorm-pants commit: failure", + "stackstorm-pants rolling report: failure" + ], + "evidence_manifest_sha256": "ec94c08b8d5b64fa278e36cb795b9fac3c01f887411aeada029a317606e965de" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1f27ac2b7ec3728c03cc60e99170085d91fe64ae531425a6ef555a8045377542", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.14576035199999637, + "build_seconds": 9.821776771000003, + "workflow_seconds": null, + "total_seconds": 9.967537123, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.36232383699999104, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.967537123 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323169828", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174491", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323169828-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323169828" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/report.md b/results/stackstorm-pants/cadence-rolling-advance/report.md new file mode 100644 index 000000000..1994e1ba1 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# stackstorm-pants: cadence-rolling-advance + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323169828 failed completion checks: + +- Workflow concluded failure +- bazel-remote stackstorm-pants commit: failure +- stackstorm-pants rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.14576035199999637 | 9.821776771000003 | 9.967537123 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..5c330abc9 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1f27ac2b7ec3728c03cc60e99170085d91fe64ae531425a6ef555a8045377542", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.14576035199999637, + "build_seconds": 9.821776771000003, + "workflow_seconds": null, + "total_seconds": 9.967537123, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.36232383699999104, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.967537123 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323169828", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174491", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323169828-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323169828" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/series.json b/results/stackstorm-pants/cadence-rolling-advance/series.json new file mode 100644 index 000000000..b7c7e73d5 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "1f27ac2b7ec3728c03cc60e99170085d91fe64ae531425a6ef555a8045377542", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "cbd4ab69f17a3ca86a1858a9378069d9fe3b8a14bdfe725bc465c02b03b44f5f" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json b/results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json new file mode 100644 index 000000000..708d0d6aa --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324025714", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "31be84e9a39d3b291d98a29e7b454c5001a2c6c6fae53ce2590634ba0ba9968c" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..38bad27fb --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:19Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324025714, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/report.json b/results/stackstorm-pants/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..8dae7d774 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "dccfdc29d50f268b3dd983b28b87c7941a08d5d6532b8d453b08efde42be3a12", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324025714", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "31be84e9a39d3b291d98a29e7b454c5001a2c6c6fae53ce2590634ba0ba9968c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10118893100001003, + "build_seconds": 28.556505596999983, + "workflow_seconds": null, + "total_seconds": 28.657694527999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0032910459999868635, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.657694527999993 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174530", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2008419569999944, + "build_seconds": 17.43477562000001, + "workflow_seconds": null, + "total_seconds": 17.635617577000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6462549910000064, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 17.635617577000005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174531", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/report.md b/results/stackstorm-pants/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..4a87bd6b0 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# stackstorm-pants: cadence-rolling-seed-02 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10118893100001003 | 28.556505596999983 | 28.657694527999993 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.2008419569999944 | 17.43477562000001 | 17.635617577000005 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..3a0acc496 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10118893100001003, + "build_seconds": 28.556505596999983, + "workflow_seconds": null, + "total_seconds": 28.657694527999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0032910459999868635, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.657694527999993 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174530", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..c8922c266 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2008419569999944, + "build_seconds": 17.43477562000001, + "workflow_seconds": null, + "total_seconds": 17.635617577000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6462549910000064, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 17.635617577000005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174531", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/series.json b/results/stackstorm-pants/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..4aa0d4b33 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "dccfdc29d50f268b3dd983b28b87c7941a08d5d6532b8d453b08efde42be3a12" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json b/results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json new file mode 100644 index 000000000..4d9b4d5f8 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322630658", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322630658", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0f464c2e7d5b9cf85c01cef28c2ef7b82927a419ece833446ecdaeafa9fb59da" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..8aee191c0 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:53Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322630658, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322630658" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/report.json b/results/stackstorm-pants/cadence-rolling-seed/report.json new file mode 100644 index 000000000..c37577793 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "bc845e5486026ab37d50479ebbfd7a316acb01f5870abd1b0254c58c4612e220", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322630658", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322630658", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0f464c2e7d5b9cf85c01cef28c2ef7b82927a419ece833446ecdaeafa9fb59da" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107338099999197, + "build_seconds": 28.043372379000004, + "workflow_seconds": null, + "total_seconds": 28.144445759999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003087786999998343, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.144445759999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174448", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135293900000306, + "build_seconds": 26.938155466000012, + "workflow_seconds": null, + "total_seconds": 27.139508405000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6489476249999484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.139508405000015 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174449", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/report.md b/results/stackstorm-pants/cadence-rolling-seed/report.md new file mode 100644 index 000000000..ca27a5fc9 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# stackstorm-pants: cadence-rolling-seed + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10107338099999197 | 28.043372379000004 | 28.144445759999996 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20135293900000306 | 26.938155466000012 | 27.139508405000015 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6aae4471c --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107338099999197, + "build_seconds": 28.043372379000004, + "workflow_seconds": null, + "total_seconds": 28.144445759999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003087786999998343, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.144445759999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174448", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..250267a68 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135293900000306, + "build_seconds": 26.938155466000012, + "workflow_seconds": null, + "total_seconds": 27.139508405000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6489476249999484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.139508405000015 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174449", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/series.json b/results/stackstorm-pants/cadence-rolling-seed/series.json new file mode 100644 index 000000000..3f30d586a --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "bc845e5486026ab37d50479ebbfd7a316acb01f5870abd1b0254c58c4612e220" +} diff --git a/results/zed-nix/cadence-fresh-01/completions/37320869828.json b/results/zed-nix/cadence-fresh-01/completions/37320869828.json new file mode 100644 index 000000000..bd6642e11 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/completions/37320869828.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320869828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320869828", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "62d6023079015ea03d9f488012cd18237a46f64bd5d88cef4fba3dbefd7d3033" +} diff --git a/results/zed-nix/cadence-fresh-01/completions/37327069343.json b/results/zed-nix/cadence-fresh-01/completions/37327069343.json new file mode 100644 index 000000000..a26b759ad --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/completions/37327069343.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37327069343", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37327069343", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "658d34d890ec9f6a5e8a11dfb1e8350fc40db343c73e8c4752a304d467e26f6a" +} diff --git a/results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json b/results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..36725eb6b --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:43:54Z", + "ref": "cadence-fresh-qualification", + "state": "requested", + "run_id": 37327069343, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37327069343" +} diff --git a/results/zed-nix/cadence-fresh-01/report.json b/results/zed-nix/cadence-fresh-01/report.json new file mode 100644 index 000000000..caceb26b6 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/report.json @@ -0,0 +1,1486 @@ +{ + "schema_version": 1, + "plan_sha256": "97e97ded96f1686fba28581b92a015bace71e0fcaf38d7ee82235e290ab4f7cd", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320869828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320869828", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "62d6023079015ea03d9f488012cd18237a46f64bd5d88cef4fba3dbefd7d3033" + }, + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37327069343", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37327069343", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "658d34d890ec9f6a5e8a11dfb1e8350fc40db343c73e8c4752a304d467e26f6a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 2823.3084382041816, + "min": 2792.4297531088596, + "max": 2854.1871232995036 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 9.83641568732537, + "min": 8.955559792726945, + "max": 10.717271581923796 + }, + "storage": { + "median": 576291607.0, + "min": 576291607, + "max": 576291607 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 2, + "measurement": { + "median": 2839.42401810587, + "min": 2830.319037953921, + "max": 2848.528998257819 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "cachix", + "count": 2, + "measurement": { + "median": 5.50904610590473, + "min": 4.8552112620657795, + "max": 6.1628809497436805 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.01418137550354, + "build_seconds": 2851.172941924, + "workflow_seconds": null, + "total_seconds": 2854.1871232995036, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2854.1871232995036 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174667", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 9.198845624923706, + "build_seconds": 1.5184259570000904, + "workflow_seconds": null, + "total_seconds": 10.717271581923796, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 10.717271581923796 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174707", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.9721765518188477, + "build_seconds": 2845.556821706, + "workflow_seconds": null, + "total_seconds": 2848.528998257819, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2848.528998257819 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174668", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.497649908065796, + "build_seconds": 2.3575613539999836, + "workflow_seconds": null, + "total_seconds": 4.8552112620657795, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 4.8552112620657795 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174708", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.295579671859741, + "build_seconds": 2788.134173437, + "workflow_seconds": null, + "total_seconds": 2792.4297531088596, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2792.4297531088596 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174703", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.493867635726929, + "build_seconds": 1.4616921570000159, + "workflow_seconds": null, + "total_seconds": 8.955559792726945, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 8.955559792726945 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174728", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.5098493099212646, + "build_seconds": 2827.8091886439997, + "workflow_seconds": null, + "total_seconds": 2830.319037953921, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2830.319037953921 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174702", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.8624181747436523, + "build_seconds": 3.300462775000028, + "workflow_seconds": null, + "total_seconds": 6.1628809497436805, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.1628809497436805 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174729", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/cadence-fresh-01/report.md b/results/zed-nix/cadence-fresh-01/report.md new file mode 100644 index 000000000..958fe304d --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/report.md @@ -0,0 +1,33 @@ +# zed-nix: cadence-fresh-01 + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 2 | 2823.3084382041816 | 2792.4297531088596–2854.1871232995036 | unmeasured | 0 | +| Warm build | BoringCache | 2 | 9.83641568732537 | 8.955559792726945–10.717271581923796 | 576291607.0 | 2 | +| Cold build | Cachix | 2 | 2839.42401810587 | 2830.319037953921–2848.528998257819 | unmeasured | 0 | +| Warm build | Cachix | 2 | 5.50904610590473 | 4.8552112620657795–6.1628809497436805 | unmeasured | 0 | + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 3.01418137550354 | 2851.172941924 | 2854.1871232995036 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 9.198845624923706 | 1.5184259570000904 | 10.717271581923796 | 576291607 | boringcache-check | hit | [JSON](runs/1-boringcache-warm.json) | +| 1 | Cachix | Cold build | 2.9721765518188477 | 2845.556821706 | 2848.528998257819 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | +| 1 | Cachix | Warm build | 2.497649908065796 | 2.3575613539999836 | 4.8552112620657795 | unmeasured | unmeasured | hit | [JSON](runs/1-cachix-warm.json) | +| 2 | BoringCache | Cold build | 4.295579671859741 | 2788.134173437 | 2792.4297531088596 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 7.493867635726929 | 1.4616921570000159 | 8.955559792726945 | 576291607 | boringcache-check | hit | [JSON](runs/2-boringcache-warm.json) | +| 2 | Cachix | Cold build | 2.5098493099212646 | 2827.8091886439997 | 2830.319037953921 | unmeasured | unmeasured | miss | [JSON](runs/2-cachix-cold.json) | +| 2 | Cachix | Warm build | 2.8624181747436523 | 3.300462775000028 | 6.1628809497436805 | unmeasured | unmeasured | hit | [JSON](runs/2-cachix-warm.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..84fdec209 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.01418137550354, + "build_seconds": 2851.172941924, + "workflow_seconds": null, + "total_seconds": 2854.1871232995036, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2854.1871232995036 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174667", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..b56e4bb0e --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 9.198845624923706, + "build_seconds": 1.5184259570000904, + "workflow_seconds": null, + "total_seconds": 10.717271581923796, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 10.717271581923796 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174707", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json b/results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json new file mode 100644 index 000000000..162c691dc --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.9721765518188477, + "build_seconds": 2845.556821706, + "workflow_seconds": null, + "total_seconds": 2848.528998257819, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2848.528998257819 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174668", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json b/results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json new file mode 100644 index 000000000..7fe7cf109 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.497649908065796, + "build_seconds": 2.3575613539999836, + "workflow_seconds": null, + "total_seconds": 4.8552112620657795, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 4.8552112620657795 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174708", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..82e181f4d --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.295579671859741, + "build_seconds": 2788.134173437, + "workflow_seconds": null, + "total_seconds": 2792.4297531088596, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2792.4297531088596 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174703", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..6f98a4bcc --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.493867635726929, + "build_seconds": 1.4616921570000159, + "workflow_seconds": null, + "total_seconds": 8.955559792726945, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 8.955559792726945 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174728", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json b/results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json new file mode 100644 index 000000000..db58f6946 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.5098493099212646, + "build_seconds": 2827.8091886439997, + "workflow_seconds": null, + "total_seconds": 2830.319037953921, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2830.319037953921 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174702", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json b/results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json new file mode 100644 index 000000000..6a4bc3ee9 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.8624181747436523, + "build_seconds": 3.300462775000028, + "workflow_seconds": null, + "total_seconds": 6.1628809497436805, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.1628809497436805 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174729", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-rolling-advance/completions/37323174641.json b/results/zed-nix/cadence-rolling-advance/completions/37323174641.json new file mode 100644 index 000000000..1f5003d42 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/completions/37323174641.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323174641", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d08da02269663b3a462ecefe0217d0b46a159fd754cd916534bdb08ed54b3f29" +} diff --git a/results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json b/results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..73f4dc197 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:58Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323174641, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641" +} diff --git a/results/zed-nix/cadence-rolling-advance/report.json b/results/zed-nix/cadence-rolling-advance/report.json new file mode 100644 index 000000000..087da9b32 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/report.json @@ -0,0 +1,412 @@ +{ + "schema_version": 1, + "plan_sha256": "5a98dac1491c4ace612df8a0bfcf5e2c14c0f6c1697f21dd0ddb3c65fcbebef1", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323174641", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d08da02269663b3a462ecefe0217d0b46a159fd754cd916534bdb08ed54b3f29" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 1872.8299867073292, + "min": 1872.8299867073292, + "max": 1872.8299867073292 + }, + "storage": { + "median": 576291607, + "min": 576291607, + "max": 576291607 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 1802.7406344062056, + "min": 1802.7406344062056, + "max": 1802.7406344062056 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.843063831329346, + "build_seconds": 1864.9869228759999, + "workflow_seconds": null, + "total_seconds": 1872.8299867073292, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1872.8299867073292 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174735", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1445672512054443, + "build_seconds": 1800.5960671550001, + "workflow_seconds": null, + "total_seconds": 1802.7406344062056, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1802.7406344062056 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174734", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/cadence-rolling-advance/report.md b/results/zed-nix/cadence-rolling-advance/report.md new file mode 100644 index 000000000..919d375d2 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/report.md @@ -0,0 +1,29 @@ +# zed-nix: cadence-rolling-advance + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 1872.8299867073292 | 1872.8299867073292–1872.8299867073292 | 576291607 | 1 | +| Changed-source build | Cachix | 1 | 1802.7406344062056 | 1802.7406344062056–1802.7406344062056 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 7.843063831329346 | 1864.9869228759999 | 1872.8299867073292 | 576291607 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.1445672512054443 | 1800.5960671550001 | 1802.7406344062056 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..e08151a2a --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.843063831329346, + "build_seconds": 1864.9869228759999, + "workflow_seconds": null, + "total_seconds": 1872.8299867073292, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1872.8299867073292 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174735", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] +} diff --git a/results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json b/results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json new file mode 100644 index 000000000..31c3c6f26 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1445672512054443, + "build_seconds": 1800.5960671550001, + "workflow_seconds": null, + "total_seconds": 1802.7406344062056, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1802.7406344062056 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174734", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] +} diff --git a/results/zed-nix/cadence-rolling-advance/series.json b/results/zed-nix/cadence-rolling-advance/series.json new file mode 100644 index 000000000..b3b752f74 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "variant": null, + "workflow_inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "5a98dac1491c4ace612df8a0bfcf5e2c14c0f6c1697f21dd0ddb3c65fcbebef1" +} diff --git a/results/zed-nix/cadence-rolling-seed/completions/37322636399.json b/results/zed-nix/cadence-rolling-seed/completions/37322636399.json new file mode 100644 index 000000000..c5deac6e2 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/completions/37322636399.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322636399", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "92a34678c864473869b9cad390bfbed9cc47760dc727a0ed17f43691a744855e" +} diff --git a/results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json b/results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..f1a1e113a --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:56Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322636399, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399" +} diff --git a/results/zed-nix/cadence-rolling-seed/report.json b/results/zed-nix/cadence-rolling-seed/report.json new file mode 100644 index 000000000..1e04db5d0 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/report.json @@ -0,0 +1,407 @@ +{ + "schema_version": 1, + "plan_sha256": "bf6c8078a29447c37ad71c86489ed872363058a299dcf47768b2c5e99b7c03af", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322636399", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "92a34678c864473869b9cad390bfbed9cc47760dc727a0ed17f43691a744855e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 2834.6952709450306, + "min": 2834.6952709450306, + "max": 2834.6952709450306 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 7.971168194222969, + "min": 7.971168194222969, + "max": 7.971168194222969 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.5863049030303955, + "build_seconds": 2829.1089660420002, + "workflow_seconds": null, + "total_seconds": 2834.6952709450306, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2834.6952709450306 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174670", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1795594692230225, + "build_seconds": 5.791608724999946, + "workflow_seconds": null, + "total_seconds": 7.971168194222969, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.971168194222969 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174671", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/cadence-rolling-seed/report.md b/results/zed-nix/cadence-rolling-seed/report.md new file mode 100644 index 000000000..111104937 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/report.md @@ -0,0 +1,29 @@ +# zed-nix: cadence-rolling-seed + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 2834.6952709450306 | 2834.6952709450306–2834.6952709450306 | unmeasured | 0 | +| Changed-source build | Cachix | 1 | 7.971168194222969 | 7.971168194222969–7.971168194222969 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 5.5863049030303955 | 2829.1089660420002 | 2834.6952709450306 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.1795594692230225 | 5.791608724999946 | 7.971168194222969 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f90bfdbf2 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.5863049030303955, + "build_seconds": 2829.1089660420002, + "workflow_seconds": null, + "total_seconds": 2834.6952709450306, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2834.6952709450306 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174670", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] +} diff --git a/results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json b/results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json new file mode 100644 index 000000000..cd7de979e --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1795594692230225, + "build_seconds": 5.791608724999946, + "workflow_seconds": null, + "total_seconds": 7.971168194222969, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.971168194222969 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174671", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] +} diff --git a/results/zed-nix/cadence-rolling-seed/series.json b/results/zed-nix/cadence-rolling-seed/series.json new file mode 100644 index 000000000..e9d00100d --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "variant": null, + "workflow_inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "bf6c8078a29447c37ad71c86489ed872363058a299dcf47768b2c5e99b7c03af" +} diff --git a/results/zed-nix/canary-screening-02/evidence.md b/results/zed-nix/canary-screening-02/evidence.md new file mode 100644 index 000000000..d9aa5c427 --- /dev/null +++ b/results/zed-nix/canary-screening-02/evidence.md @@ -0,0 +1,13 @@ +# Retained failed Nix screen + +[Run 37296214039](https://github.com/boringcache/benchmarks/actions/runs/37296214039) +failed both warm jobs at the dependency-baseline comparison. Both providers +restored the measured Zed package, verified its output and produced matching +package closure records. The separately built dependency closure differed +between workers. This run does not qualify the cold/warm comparison. + +The [original evidence bundle](https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37296214039.tar.gz) +includes the available run, attempts, jobs, logs, artifacts, commit and workflow. +The published archive was downloaded again, its SHA-256 matched, and all 15 +listed files and its scoped inventory passed verification without gaps. +See [the archive inventory](../../../migration/evidence.json). diff --git a/scripts/benchmark-catalog.rb b/scripts/benchmark-catalog.rb index 4ee48d44a..7c09ff4dd 100644 --- a/scripts/benchmark-catalog.rb +++ b/scripts/benchmark-catalog.rb @@ -53,7 +53,8 @@ def self.entry(directory, root:, evidence:) end archives = run_ids.filter_map { |id| evidence[[BenchmarkCases::REPOSITORY, id]] }.map do |archive| archive.slice("repository", "run_id", "inventory_state", "verified_files", "gaps", "scope", - "unavailable_workflow_evidence", "release_url", "bundle_url", "bundle_bytes", "sha256", "publication_verification") + "unavailable_workflow_evidence", "release_url", "bundle_url", "bundle_bytes", "sha256", "publication_verification", + "bundle_format", "archive_sha256", "archive_bytes", "bundle_parts") end base.merge(plan.slice("lane", "variant", "sample_count", "comparison", "source", "plan_sha256", "definition_sha256"), "state" => state, "publication" => report.fetch("publication"), diff --git a/scripts/collect-series.rb b/scripts/collect-series.rb index 03812f4f3..3c09999e5 100644 --- a/scripts/collect-series.rb +++ b/scripts/collect-series.rb @@ -26,8 +26,13 @@ def self.call(directory, evidence_directory:, sample:) names.lines.map(&:strip).grep(/\.json\z/).each do |name| # Proof bundles also contain raw product evidence. Only collect canonical # phase filenames or the contents of a dedicated phase artifact. - next unless artifact.fetch("name").start_with?("phase-") || name.match?(/-(?:fresh-(?:cold|warm)|rolling-commit)\.json\z/) + canonical_name = name.match?(/-(?:fresh-(?:cold|warm)|rolling-commit)\.json\z/) + next unless artifact.fetch("name").start_with?("phase-") || canonical_name value = JSON.parse(read_record(archive, name)) + # Some phase bundles also retain native counters and provider receipts. + # Keep those in the export without treating them as series observations. + # A canonical filename still requires the complete record contract. + next unless canonical_name || (value.is_a?(Hash) && value.key?("series")) BenchmarkSeries.validate_record(plan, value) unless value.dig("github", "run_id").to_s == run_id && value.dig("series", "sample") == sample raise BenchmarkSeries::Error, "Phase artifact belongs to another run or sample" diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index 86d65b38d..30859cb54 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -72,6 +72,10 @@ def self.rolling_seed "current" => {"case_id" => ENV.fetch("BENCHMARK_ID"), "source" => context.fetch("source"), "run_id" => ENV.fetch("GITHUB_RUN_ID"), "run_attempt" => ENV.fetch("GITHUB_RUN_ATTEMPT")}} File.write("reapi-evidence/rolling-seed.json", JSON.pretty_generate(seed) + "\n") + # bazel-remote rejects extra files in its store. Retain the source receipt + # in evidence before starting it; publish_seed writes the next receipt only + # after the server has stopped and output verification has passed. + File.delete(path) if File.file?(path) end def self.publish_seed diff --git a/scripts/source-promotion.rb b/scripts/source-promotion.rb index 1c5eb0b01..05b729d26 100644 --- a/scripts/source-promotion.rb +++ b/scripts/source-promotion.rb @@ -122,7 +122,15 @@ def commit(changes, expected:, message:) errors = result.fetch("errors", []) # Only an optimistic-concurrency conflict is safe to retry. Other API # failures may have an uncertain outcome and need their receipt inspected. - raise Error, "Source publication failed: #{errors.map { |error| error['message'] }.join('; ')}" unless errors.any? { |error| error["type"] == "STALE_DATA" } + retryable = errors.any? && errors.all? do |error| + # A ref-lock race after GraphQL's initial check is reported as + # FORBIDDEN. Retry only its exact old/new SHA response, not permission + # failures or an uncertain request outcome. + race = error["message"].to_s.match(/\Ais at ([0-9a-f]{40}) but expected #{Regexp.escape(head)}\z/) + error["type"] == "STALE_DATA" || + (error["type"] == "FORBIDDEN" && error["path"] == ["createCommitOnBranch"] && race && race[1] != head) + end + raise Error, "Source publication failed: #{errors.map { |error| error['message'] }.join('; ')}" unless retryable end raise Error, "Source publication could not acquire the current branch head" end diff --git a/scripts/verify-upstream-recipe.rb b/scripts/verify-upstream-recipe.rb index 80e20ee9b..232e9a5a2 100755 --- a/scripts/verify-upstream-recipe.rb +++ b/scripts/verify-upstream-recipe.rb @@ -12,6 +12,17 @@ if contract.key?("upstream_files_by_revision") revision, status = Open3.capture2("git", "-C", source, "rev-parse", "HEAD") raise "Cannot determine the source revision for recipe verification" unless status.success? + context_path = File.join(root, "benchmark-context.json") + if File.realpath(source) == File.realpath(root) && File.file?(context_path) + declared = JSON.parse(File.read(context_path)).dig("source", "revision") + if declared + parent, status = Open3.capture2("git", "-C", source, "rev-parse", "HEAD^") + unless declared.match?(/\A[0-9a-f]{40}\z/) && status.success? && parent.strip == declared + raise "Prepared snapshot does not have the declared upstream parent" + end + revision = declared + end + end overrides = contract.fetch("upstream_files_by_revision") raise "Recipe exceptions require exact source revisions" unless overrides.keys.all? { |sha| sha.match?(/\A[0-9a-f]{40}\z/) } files = overrides.fetch(revision.strip, files) diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 4802f38ea..4636e6258 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -318,6 +318,21 @@ def test_reviewed_recipe_rejects_changed_upstream_files end end + def test_continuous_obs_report_keeps_an_absent_cache_hit_unmeasured + with_case("obs-studio") do |directory| + FileUtils.cp(File.join(ROOT, "scripts/canonical/benchmark-report.rb"), File.join(directory, "scripts/benchmark-report.rb")) + _, error, status = run_script(directory, "write_phase_result", "--surface", "xcode", + "--strategy", "boringcache", "--phase", "rolling", "--continuous", "--cache-hit", "", + "--source-sha", "a" * 40, "--restore-seconds", "0", "--build-seconds", "3") + assert status.success?, error + native = JSON.parse(File.read(File.join(directory, "benchmark-results/xcode-boringcache-rolling.json"))) + assert_equal "unmeasured", native.dig("classification", "cache_import_status") + record = JSON.parse(File.read(File.join(directory, "benchmark-results/obs-studio-boringcache-xcode-rolling-commit.json"))) + assert_nil record.dig("cache", "hit") + assert_equal 3.0, record.dig("timing", "build_seconds") + end + end + def test_selected_recipe_contract_checks_each_declared_plan with_case("zed") do |directory| source = File.join(directory, "upstream") @@ -373,6 +388,31 @@ def test_parent_recipe_exception_is_limited_to_its_exact_revision_and_file_set end end + def test_prepared_snapshot_recipe_uses_only_its_verified_upstream_parent + with_case("hugo-go") do |directory| + git = ->(*args) { Open3.capture2e("git", "-C", directory, "-c", "user.name=Benchmark test", "-c", "user.email=benchmark@localhost", "-c", "commit.gpgsign=false", *args) } + git.call("init") + File.write(File.join(directory, "recipe.txt"), "parent recipe\n") + git.call("add", ".") + _, status = git.call("commit", "-m", "Upstream source") + assert status.success? + revision, = git.call("rev-parse", "HEAD") + contract = {"upstream_files" => {"recipe.txt" => Digest::SHA256.hexdigest("new recipe\n")}, + "upstream_files_by_revision" => {revision.strip => {"recipe.txt" => Digest::SHA256.hexdigest("parent recipe\n")}}, "commands" => {}} + File.write(File.join(directory, "recipe-contract.json"), JSON.generate(contract)) + git.call("add", ".") + git.call("commit", "-m", "Prepare workload") + context = File.join(directory, "benchmark-context.json") + File.write(context, JSON.generate({"source" => {"revision" => revision.strip}})) + _, error, status = run_script(directory, "verify-upstream-recipe", directory) + assert status.success?, error + git.call("commit", "--allow-empty", "-m", "Unexpected wrapper") + _, error, status = run_script(directory, "verify-upstream-recipe", directory) + refute status.success? + assert_includes error, "does not have the declared upstream parent" + end + end + def test_deno_profile_selects_exactly_one_declared_profile with_case("deno") do |directory| path = File.join(directory, ".boringcache.toml") diff --git a/test/benchmark_catalog_test.rb b/test/benchmark_catalog_test.rb index 0f9dc83cd..0bc393317 100644 --- a/test/benchmark_catalog_test.rb +++ b/test/benchmark_catalog_test.rb @@ -97,6 +97,8 @@ def test_failed_completions_and_archive_gaps_remain_visible_without_observations completion(run_id: "12", status: "failed") archive = {"repository" => BenchmarkCases::REPOSITORY, "run_id" => 12, "inventory_state" => "complete", "release_url" => "https://github.com/boringcache/benchmarks/releases/tag/evidence-example", + "bundle_format" => "split-tar-gzip-index", "archive_sha256" => "f" * 64, "archive_bytes" => 3_000_000_000, + "bundle_parts" => [{"name" => "evidence.tar.gz.part001", "sha256" => "e" * 64, "bytes" => 1_900_000_000}], "scope" => "Available logs only", "unavailable_workflow_evidence" => ["Original phase JSON was never uploaded"]} BenchmarkCases.write_json(File.join(@root, "migration/evidence.json"), {"exports" => [archive]}) result = entry diff --git a/test/benchmark_series_test.rb b/test/benchmark_series_test.rb index d01cc86a8..8bae32d61 100644 --- a/test/benchmark_series_test.rb +++ b/test/benchmark_series_test.rb @@ -208,6 +208,24 @@ def test_proof_bundle_records_still_require_matching_case_and_dispatch assert_empty Dir[File.join(@directory, "runs", "*.json")] end + def test_phase_bundles_can_retain_native_counters_alongside_the_canonical_record + value = record + directory = phase_export([value], artifact_name: "phase-example-boringcache-release-rolling-commit", + filenames: ["example-boringcache-release-rolling-commit.json"], + extra_files: {"ccache-native.json" => {"direct_cache_hit" => 8}, + "ccache-rolling.json" => {"schema_version" => 2, "timing" => {"build_seconds" => 3}}}) + result = CollectSeries.call(@directory, evidence_directory: directory, sample: 1) + assert_equal 1, result["records"] + assert_equal value, JSON.parse(File.read(File.join(@directory, "runs", "1-boringcache-commit.json"))) + end + + def test_canonical_phase_filename_cannot_hide_a_missing_series_contract + directory = phase_export([record.except("series")], + filenames: ["example-boringcache-release-rolling-commit.json"]) + assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } + assert_empty Dir[File.join(@directory, "runs", "*.json")] + end + def test_collection_requires_the_original_dispatch_and_matching_run directory = phase_export([record.merge("github" => {"run_id" => "2"})]) assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } diff --git a/test/reapi_setup_test.rb b/test/reapi_setup_test.rb new file mode 100644 index 000000000..ed6a6a06f --- /dev/null +++ b/test/reapi_setup_test.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "tmpdir" +require_relative "../scripts/reapi-setup" + +class ReapiSetupTest < Minitest::Test + def test_restored_receipt_is_retained_outside_the_running_store + with_store do |previous| + ReapiSetup.rolling_seed + refute File.exist?("reapi-store/benchmark-source.json") + assert_equal "cached object", File.read("reapi-store/cas/object") + evidence = JSON.parse(File.read("reapi-evidence/rolling-seed.json")) + assert_equal previous, evidence.fetch("previous") + assert_equal "seed-key", evidence.fetch("cache_key") + ReapiSetup.publish_seed + next_seed = JSON.parse(File.read("reapi-store/benchmark-source.json")) + assert_equal "b" * 40, next_seed.dig("source", "revision") + assert_equal "2", next_seed.fetch("run_id") + assert_equal "cached object", File.read("reapi-store/cas/object") + end + end + + def test_foreign_seed_is_rejected_without_discarding_its_receipt + with_store do |previous| + previous["case_id"] = "another-case" + File.write("reapi-store/benchmark-source.json", JSON.generate(previous)) + error = assert_raises(RuntimeError) { ReapiSetup.rolling_seed } + assert_includes error.message, "belongs to another case" + assert_equal previous, JSON.parse(File.read("reapi-store/benchmark-source.json")) + refute File.exist?("reapi-evidence/rolling-seed.json") + end + end + + private + + def with_store + original = ENV.to_h + ENV.update("BENCHMARK_ID" => "gogs-moon", "RESTORED_CACHE_KEY" => "seed-key", "GITHUB_RUN_ID" => "2", "GITHUB_RUN_ATTEMPT" => "1") + Dir.mktmpdir do |directory| + Dir.chdir(directory) do + FileUtils.mkdir_p(%w[reapi-store/cas reapi-evidence]) + File.write("reapi-store/cas/object", "cached object") + previous = {"case_id" => "gogs-moon", "source" => {"revision" => "a" * 40}, "run_id" => "1"} + File.write("reapi-store/benchmark-source.json", JSON.generate(previous)) + File.write("benchmark-context.json", JSON.generate("source" => {"revision" => "b" * 40})) + yield previous + end + end + ensure + ENV.replace(original) + end +end diff --git a/test/source_promotion_test.rb b/test/source_promotion_test.rb index bc18da236..67e99581b 100644 --- a/test/source_promotion_test.rb +++ b/test/source_promotion_test.rb @@ -160,13 +160,42 @@ def test_publication_rejects_a_concurrent_edit_instead_of_overwriting_it end def test_unrelated_branch_advancement_retries_with_a_new_expected_head + [ + {"type" => "STALE_DATA", "message" => "Branch advanced"}, + {"type" => "FORBIDDEN", "path" => ["createCommitOnBranch"], "message" => "is at #{'c' * 40} but expected #{'b' * 40}"} + ].each { |conflict| assert_concurrent_commit_retries(conflict) } + end + + def test_permission_failures_and_unrelated_ref_errors_are_not_retried + ["Resource not accessible by integration", "is at #{'c' * 40} but expected #{'a' * 40}"].each do |message| + publisher = SourcePromotion::Publisher.new + requests = 0 + api = lambda do |path, body: nil| + if path == "graphql" + requests += 1 + {"errors" => [{"type" => "FORBIDDEN", "path" => ["createCommitOnBranch"], "message" => message}]} + else + {"object" => {"sha" => "b" * 40}} + end + end + publisher.stub(:api, api) do + error = assert_raises(SourcePromotion::Error) { publisher.commit({"receipt.json" => "{}"}, expected: {}, message: "Record receipt") } + assert_includes error.message, message + end + assert_equal 1, requests + end + end + + private + + def assert_concurrent_commit_retries(conflict) publisher = SourcePromotion::Publisher.new heads = ["b" * 40, "c" * 40] commits = [] api = lambda do |path, body: nil| if path == "graphql" commits << body.fetch("variables").fetch("input") - commits.length == 1 ? {"errors" => [{"type" => "STALE_DATA", "message" => "Branch advanced"}]} : {"data" => {"createCommitOnBranch" => {"commit" => {"oid" => "d" * 40}}}} + commits.length == 1 ? {"errors" => [conflict]} : {"data" => {"createCommitOnBranch" => {"commit" => {"oid" => "d" * 40}}}} elsif path.include?("git/ref") {"object" => {"sha" => heads.shift}} else