From be73c94ef9373756b2eb0b676dd6a26b7b6ac842 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:11:20 +0100 Subject: [PATCH 01/12] Verify prepared snapshot recipes against their upstream parent --- migration/evidence.json | 14 +++++++++++ .../zed-nix/canary-screening-02/evidence.md | 13 ++++++++++ scripts/verify-upstream-recipe.rb | 11 ++++++++ test/benchmark_case_scripts_test.rb | 25 +++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 results/zed-nix/canary-screening-02/evidence.md diff --git a/migration/evidence.json b/migration/evidence.json index 58bba729d..cf2059a76 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -16,6 +16,20 @@ "partial": 1 }, "unavailable_reference_count": 1 + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37296214039, + "verified_files": 15, + "gaps": [], + "sha256": "02cedef26b2bf4a6e4fd58b6ba454be3d6ea10e8a2daf6c08d92e511abdb5a67", + "bundle_bytes": 6100745, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37296214039.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This failed run, including available attempts, jobs, logs, artifacts, commit and workflow" } ], "exports": [ diff --git a/results/zed-nix/canary-screening-02/evidence.md b/results/zed-nix/canary-screening-02/evidence.md new file mode 100644 index 000000000..d9aa5c427 --- /dev/null +++ b/results/zed-nix/canary-screening-02/evidence.md @@ -0,0 +1,13 @@ +# Retained failed Nix screen + +[Run 37296214039](https://github.com/boringcache/benchmarks/actions/runs/37296214039) +failed both warm jobs at the dependency-baseline comparison. Both providers +restored the measured Zed package, verified its output and produced matching +package closure records. The separately built dependency closure differed +between workers. This run does not qualify the cold/warm comparison. + +The [original evidence bundle](https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37296214039.tar.gz) +includes the available run, attempts, jobs, logs, artifacts, commit and workflow. +The published archive was downloaded again, its SHA-256 matched, and all 15 +listed files and its scoped inventory passed verification without gaps. +See [the archive inventory](../../../migration/evidence.json). diff --git a/scripts/verify-upstream-recipe.rb b/scripts/verify-upstream-recipe.rb index 80e20ee9b..232e9a5a2 100755 --- a/scripts/verify-upstream-recipe.rb +++ b/scripts/verify-upstream-recipe.rb @@ -12,6 +12,17 @@ if contract.key?("upstream_files_by_revision") revision, status = Open3.capture2("git", "-C", source, "rev-parse", "HEAD") raise "Cannot determine the source revision for recipe verification" unless status.success? + context_path = File.join(root, "benchmark-context.json") + if File.realpath(source) == File.realpath(root) && File.file?(context_path) + declared = JSON.parse(File.read(context_path)).dig("source", "revision") + if declared + parent, status = Open3.capture2("git", "-C", source, "rev-parse", "HEAD^") + unless declared.match?(/\A[0-9a-f]{40}\z/) && status.success? && parent.strip == declared + raise "Prepared snapshot does not have the declared upstream parent" + end + revision = declared + end + end overrides = contract.fetch("upstream_files_by_revision") raise "Recipe exceptions require exact source revisions" unless overrides.keys.all? { |sha| sha.match?(/\A[0-9a-f]{40}\z/) } files = overrides.fetch(revision.strip, files) diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 4802f38ea..3e37df527 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -373,6 +373,31 @@ def test_parent_recipe_exception_is_limited_to_its_exact_revision_and_file_set end end + def test_prepared_snapshot_recipe_uses_only_its_verified_upstream_parent + with_case("hugo-go") do |directory| + git = ->(*args) { Open3.capture2e("git", "-C", directory, "-c", "user.name=Benchmark test", "-c", "user.email=benchmark@localhost", "-c", "commit.gpgsign=false", *args) } + git.call("init") + File.write(File.join(directory, "recipe.txt"), "parent recipe\n") + git.call("add", ".") + _, status = git.call("commit", "-m", "Upstream source") + assert status.success? + revision, = git.call("rev-parse", "HEAD") + contract = {"upstream_files" => {"recipe.txt" => Digest::SHA256.hexdigest("new recipe\n")}, + "upstream_files_by_revision" => {revision.strip => {"recipe.txt" => Digest::SHA256.hexdigest("parent recipe\n")}}, "commands" => {}} + File.write(File.join(directory, "recipe-contract.json"), JSON.generate(contract)) + git.call("add", ".") + git.call("commit", "-m", "Prepare workload") + context = File.join(directory, "benchmark-context.json") + File.write(context, JSON.generate({"source" => {"revision" => revision.strip}})) + _, error, status = run_script(directory, "verify-upstream-recipe", directory) + assert status.success?, error + git.call("commit", "--allow-empty", "-m", "Unexpected wrapper") + _, error, status = run_script(directory, "verify-upstream-recipe", directory) + refute status.success? + assert_includes error, "does not have the declared upstream parent" + end + end + def test_deno_profile_selects_exactly_one_declared_profile with_case("deno") do |directory| path = File.join(directory, ".boringcache.toml") From 2b49254f3b5da21eb6516462e7a61feec42464bd Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:15:27 +0100 Subject: [PATCH 02/12] Install the ccache version required by the canary CLI --- cases/obs-studio/payload/scripts/install-ccache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cases/obs-studio/payload/scripts/install-ccache.sh b/cases/obs-studio/payload/scripts/install-ccache.sh index b0420e097..6356084b8 100755 --- a/cases/obs-studio/payload/scripts/install-ccache.sh +++ b/cases/obs-studio/payload/scripts/install-ccache.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -ccache_version=4.14 +ccache_version=4.14.1 ccache_archive="ccache-${ccache_version}-linux-x86_64-glibc.tar.gz" -ccache_sha256=c64760b0b85ba86068f4cd162dc42e2dc39c6f46b0cb8c1990dfccbec7a1fed0 +ccache_sha256=67f8b4cd2620b0d7bc9e834b3baea76bcdca4f2f3f47fe658c343fee58f816fa ccache_url="https://github.com/ccache/ccache/releases/download/v${ccache_version}/${ccache_archive}" storage_version=0.10 From 1a9a35527c47034db1f4399c36253cb6e9516b39 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:20:51 +0100 Subject: [PATCH 03/12] Remove retained seed metadata before starting bazel-remote --- .github/workflows/source-case.yml | 6 ++-- scripts/reapi-setup.rb | 4 +++ test/reapi_setup_test.rb | 53 +++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 test/reapi_setup_test.rb diff --git a/.github/workflows/source-case.yml b/.github/workflows/source-case.yml index bf9cef333..e40b525e3 100644 --- a/.github/workflows/source-case.yml +++ b/.github/workflows/source-case.yml @@ -10,7 +10,7 @@ permissions: actions: read jobs: previous: - name: '${{ inputs.case_id }} previous rolling run' + name: Reconcile previous runs runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -48,7 +48,7 @@ jobs: if-no-files-found: ignore retention-days: 90 inspect: - name: '${{ inputs.case_id }} source check' + name: Check source needs: previous if: needs.previous.outputs.proceed == 'true' runs-on: ubuntu-24.04 @@ -87,7 +87,7 @@ jobs: if-no-files-found: error retention-days: 90 publish: - name: '${{ inputs.case_id }} queue rolling runs' + name: Prepare rolling runs needs: [previous, inspect] if: needs.inspect.outputs.changed == 'true' runs-on: ubuntu-24.04 diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index 86d65b38d..30859cb54 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -72,6 +72,10 @@ def self.rolling_seed "current" => {"case_id" => ENV.fetch("BENCHMARK_ID"), "source" => context.fetch("source"), "run_id" => ENV.fetch("GITHUB_RUN_ID"), "run_attempt" => ENV.fetch("GITHUB_RUN_ATTEMPT")}} File.write("reapi-evidence/rolling-seed.json", JSON.pretty_generate(seed) + "\n") + # bazel-remote rejects extra files in its store. Retain the source receipt + # in evidence before starting it; publish_seed writes the next receipt only + # after the server has stopped and output verification has passed. + File.delete(path) if File.file?(path) end def self.publish_seed diff --git a/test/reapi_setup_test.rb b/test/reapi_setup_test.rb new file mode 100644 index 000000000..ed6a6a06f --- /dev/null +++ b/test/reapi_setup_test.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "tmpdir" +require_relative "../scripts/reapi-setup" + +class ReapiSetupTest < Minitest::Test + def test_restored_receipt_is_retained_outside_the_running_store + with_store do |previous| + ReapiSetup.rolling_seed + refute File.exist?("reapi-store/benchmark-source.json") + assert_equal "cached object", File.read("reapi-store/cas/object") + evidence = JSON.parse(File.read("reapi-evidence/rolling-seed.json")) + assert_equal previous, evidence.fetch("previous") + assert_equal "seed-key", evidence.fetch("cache_key") + ReapiSetup.publish_seed + next_seed = JSON.parse(File.read("reapi-store/benchmark-source.json")) + assert_equal "b" * 40, next_seed.dig("source", "revision") + assert_equal "2", next_seed.fetch("run_id") + assert_equal "cached object", File.read("reapi-store/cas/object") + end + end + + def test_foreign_seed_is_rejected_without_discarding_its_receipt + with_store do |previous| + previous["case_id"] = "another-case" + File.write("reapi-store/benchmark-source.json", JSON.generate(previous)) + error = assert_raises(RuntimeError) { ReapiSetup.rolling_seed } + assert_includes error.message, "belongs to another case" + assert_equal previous, JSON.parse(File.read("reapi-store/benchmark-source.json")) + refute File.exist?("reapi-evidence/rolling-seed.json") + end + end + + private + + def with_store + original = ENV.to_h + ENV.update("BENCHMARK_ID" => "gogs-moon", "RESTORED_CACHE_KEY" => "seed-key", "GITHUB_RUN_ID" => "2", "GITHUB_RUN_ATTEMPT" => "1") + Dir.mktmpdir do |directory| + Dir.chdir(directory) do + FileUtils.mkdir_p(%w[reapi-store/cas reapi-evidence]) + File.write("reapi-store/cas/object", "cached object") + previous = {"case_id" => "gogs-moon", "source" => {"revision" => "a" * 40}, "run_id" => "1"} + File.write("reapi-store/benchmark-source.json", JSON.generate(previous)) + File.write("benchmark-context.json", JSON.generate("source" => {"revision" => "b" * 40})) + yield previous + end + end + ensure + ENV.replace(original) + end +end From 350feb68b6fe6ec9ef75cfbe07ca6af58366370c Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:32:09 +0100 Subject: [PATCH 04/12] Prepare historical schedule cutover without activating it --- docs/cadence-cutover.md | 45 ++ docs/cadence.md | 10 +- .../benchmark-chroma.patch | 34 + .../benchmark-deno.patch | 23 + .../benchmark-duckgres.patch | 34 + .../benchmark-grpc.patch | 33 + .../benchmark-hugo-go.patch | 34 + .../benchmark-hugo.patch | 33 + .../benchmark-immich.patch | 33 + .../benchmark-linkerd2.patch | 33 + .../benchmark-mastodon.patch | 33 + .../benchmark-n8n.patch | 45 ++ .../benchmark-obs-studio.patch | 12 + .../benchmark-opentelemetry-java.patch | 34 + .../benchmark-posthog.patch | 34 + .../benchmark-qdrant.patch | 33 + .../benchmark-spring-ai.patch | 34 + .../benchmark-storybook.patch | 34 + .../benchmark-zed.patch | 23 + ...cadence-schedule-inventory-2026-10-05.json | 765 ++++++++++++++++++ 20 files changed, 1356 insertions(+), 3 deletions(-) create mode 100644 docs/cadence-cutover.md create mode 100644 migration/cadence-cutover-patches/benchmark-chroma.patch create mode 100644 migration/cadence-cutover-patches/benchmark-deno.patch create mode 100644 migration/cadence-cutover-patches/benchmark-duckgres.patch create mode 100644 migration/cadence-cutover-patches/benchmark-grpc.patch create mode 100644 migration/cadence-cutover-patches/benchmark-hugo-go.patch create mode 100644 migration/cadence-cutover-patches/benchmark-hugo.patch create mode 100644 migration/cadence-cutover-patches/benchmark-immich.patch create mode 100644 migration/cadence-cutover-patches/benchmark-linkerd2.patch create mode 100644 migration/cadence-cutover-patches/benchmark-mastodon.patch create mode 100644 migration/cadence-cutover-patches/benchmark-n8n.patch create mode 100644 migration/cadence-cutover-patches/benchmark-obs-studio.patch create mode 100644 migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch create mode 100644 migration/cadence-cutover-patches/benchmark-posthog.patch create mode 100644 migration/cadence-cutover-patches/benchmark-qdrant.patch create mode 100644 migration/cadence-cutover-patches/benchmark-spring-ai.patch create mode 100644 migration/cadence-cutover-patches/benchmark-storybook.patch create mode 100644 migration/cadence-cutover-patches/benchmark-zed.patch create mode 100644 migration/cadence-schedule-inventory-2026-10-05.json diff --git a/docs/cadence-cutover.md b/docs/cadence-cutover.md new file mode 100644 index 000000000..ccceafe5d --- /dev/null +++ b/docs/cadence-cutover.md @@ -0,0 +1,45 @@ +# Cadence cutover + +The new workflows are merged, but central activation is off. The user deferred +activation until a compatible stable CLI is released. Historical cron triggers +remain active. Do not apply the prepared patches or set the activation variable +before the release and qualification gates in [cadence.md](cadence.md) pass. + +The [October 5 inventory](../migration/cadence-schedule-inventory-2026-10-05.json) +records 48 scheduled workflows in the 17 historical repositories represented by +the maintained suite. It records the default branch, observed head, workflow +blob SHA, cron expressions and remaining entrypoints. The seven snapshot cases +have no replaced BoringCache repository schedules. + +[Prepared patches](../migration/cadence-cutover-patches) remove only each +inventoried workflow's `schedule` event. Each edited document was parsed and +compared with the original after removing that event; manual and PR events, jobs, +permissions and other workflow behavior remain equivalent. These patches have +not been applied. Unselected repositories, including Docker corpus, are outside +this cutover. + +When release and qualification are complete: + +1. Resolve the exact stable CLI and run the full 31-target dry run on `main`. + Require the REAPI capability check to pass without omissions or substitution + of a canary. Confirm hosted checks and the retained qualification records. +2. Refresh the historical inventory. Review any changed workflow blobs before + applying their patches. Keep central activation off while removing all 48 + historical cron triggers. Preserve their manual and PR entrypoints. +3. Query every historical repository for queued, pending and running scheduled + or source-dispatched work. Allow those runs and downstream requests to finish; + retain their run IDs. Recheck that no replaced cron triggers remain. Do not + cancel unrelated runs or treat a stopped scheduler as a drained queue. +4. Set `BENCHMARK_CADENCE_ACTIVE=true` in `boringcache/benchmarks`. This transfers + source checks, weekly/nightly dispatch and canary monitoring to the central + suite. The first hourly source check should produce per-case observations; + changed cases should retain intent and requested run IDs independently. +5. Verify the first complete source cycle and the first weekly and nightly + dispatches. Compare each receipt with the 24-case / 31-fresh-target inventory, + then inspect leaf outcomes and comparison reports. Request acceptance alone + is not build completion. + +If central ownership must be stopped, unset the activation variable first and +inspect already requested central runs. Restore historical cron triggers only +after central requests have drained, so both owners do not request the same +work. Retain source receipts and observations during either transition. diff --git a/docs/cadence.md b/docs/cadence.md index 7f891dcbb..bec67f2bf 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -1,10 +1,12 @@ # Benchmark schedules -Central activation remains off. Historical repositories still own the active +PR #42 is merged. Central activation remains off. Historical repositories still own the active weekly, canary and source schedules. Central scheduled jobs require `BENCHMARK_CADENCE_ACTIVE=true`; merging this branch does not activate them. Manual qualification remains available. Historical triggers must be retired -before ownership changes, so the migration does not duplicate runs. +before ownership changes, so the migration does not duplicate runs. The +[cutover procedure](cadence-cutover.md) includes the observed inventory and +prepared patches for 48 cron triggers in 17 repositories; none are applied yet. `suites/scheduled.json` selects the maintained suite plus Moon, Pants, Buck2, sbt and Nix: 24 source cases and 31 fresh targets. It excludes Docker corpus and @@ -83,7 +85,9 @@ The October 5 preflight rejected stable `v1.33.0`: it does not expose 31-target preflight. Weekly activation must wait for a compatible stable release; it must not substitute a canary or silently omit the REAPI cases. -The first hosted source matrix completed the cases independently and identified +The complete controller rehearsal passed all 24 cases and planned changed-source +rolling requests with the canary without publishing or dispatching builds. +The earlier source matrix completed the cases independently and identified recipe changes in Immich, Qdrant, msgpack and Zed. Their reviewed changes are recorded in [the recipe review](recipe-reviews-2026-10-05.md). New source pins and changed recipes still require live qualification. diff --git a/migration/cadence-cutover-patches/benchmark-chroma.patch b/migration/cadence-cutover-patches/benchmark-chroma.patch new file mode 100644 index 000000000..ed6a0a445 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-chroma.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "2 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/chroma-fresh-benchmark.yml ++++ b/.github/workflows/chroma-fresh-benchmark.yml +@@ -25,8 +25,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 1" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "1,31 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-deno.patch b/migration/cadence-cutover-patches/benchmark-deno.patch new file mode 100644 index 000000000..895a0bf50 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-deno.patch @@ -0,0 +1,23 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "3 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "2,32 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-duckgres.patch b/migration/cadence-cutover-patches/benchmark-duckgres.patch new file mode 100644 index 000000000..a7070487a --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-duckgres.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "5 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/duckgres-fresh-benchmark.yml ++++ b/.github/workflows/duckgres-fresh-benchmark.yml +@@ -8,8 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: +- - cron: "0 3 * * 1" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "5,35 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-grpc.patch b/migration/cadence-cutover-patches/benchmark-grpc.patch new file mode 100644 index 000000000..4b2797517 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-grpc.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "7 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/grpc-bazel-fresh-benchmark.yml ++++ b/.github/workflows/grpc-bazel-fresh-benchmark.yml +@@ -7,7 +7,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 5"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/grpc-bazel-benchmark/**", ".github/workflows/grpc-bazel-benchmark.yml", ".github/workflows/grpc-bazel-fresh-benchmark.yml", "scripts/**", "upstream"] + permissions: {contents: read, actions: write} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "7,37 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-hugo-go.patch b/migration/cadence-cutover-patches/benchmark-hugo-go.patch new file mode 100644 index 000000000..4e5ba1bed --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-hugo-go.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "11 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/hugo-go-fresh-benchmark.yml ++++ b/.github/workflows/hugo-go-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 4 * * 1" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "10,40 * * * *" +- + permissions: + contents: write + actions: write diff --git a/migration/cadence-cutover-patches/benchmark-hugo.patch b/migration/cadence-cutover-patches/benchmark-hugo.patch new file mode 100644 index 000000000..43222ce92 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-hugo.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "9 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/hugo-fresh-benchmark.yml ++++ b/.github/workflows/hugo-fresh-benchmark.yml +@@ -9,7 +9,6 @@ + cli_platform: {description: "Optional BoringCache CLI platform.", required: false, type: string, default: linux-amd64} + benchmark_id_suffix: {description: "Optional suffix for an isolated comparison run.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 1"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/hugo-docker-benchmark/**", ".github/workflows/hugo-benchmark.yml", ".github/workflows/hugo-fresh-benchmark.yml", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "9,39 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-immich.patch b/migration/cadence-cutover-patches/benchmark-immich.patch new file mode 100644 index 000000000..0db25ab76 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-immich.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "12 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/immich-fresh-benchmark.yml ++++ b/.github/workflows/immich-fresh-benchmark.yml +@@ -8,7 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 2"}] + pull_request: + paths: [".gitmodules", ".github/actions/immich-docker-benchmark/**", ".github/workflows/immich-benchmark.yml", ".github/workflows/immich-fresh-benchmark.yml", "libjxl-patches/**", "scenarios/**", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "12,42 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-linkerd2.patch b/migration/cadence-cutover-patches/benchmark-linkerd2.patch new file mode 100644 index 000000000..970bf57fb --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-linkerd2.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "14 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/linkerd2-fresh-benchmark.yml ++++ b/.github/workflows/linkerd2-fresh-benchmark.yml +@@ -8,7 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 1"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/linkerd2-docker-benchmark/**", ".github/workflows/linkerd2-benchmark.yml", ".github/workflows/linkerd2-fresh-benchmark.yml", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "13,43 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-mastodon.patch b/migration/cadence-cutover-patches/benchmark-mastodon.patch new file mode 100644 index 000000000..3873ad8f3 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-mastodon.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "16 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/mastodon-fresh-benchmark.yml ++++ b/.github/workflows/mastodon-fresh-benchmark.yml +@@ -9,7 +9,6 @@ + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + upstream_ref: {description: "Optional Mastodon ref or SHA instead of the pinned submodule commit.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 4 * * 2"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/mastodon-docker-benchmark/**", ".github/workflows/mastodon-docker-benchmark.yml", ".github/workflows/mastodon-fresh-benchmark.yml", "scenarios/**", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "15,45 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-n8n.patch b/migration/cadence-cutover-patches/benchmark-n8n.patch new file mode 100644 index 000000000..5816812b8 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-n8n.patch @@ -0,0 +1,45 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "18 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/n8n-docker-fresh-benchmark.yml ++++ b/.github/workflows/n8n-docker-fresh-benchmark.yml +@@ -25,8 +25,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "30 3 * * 6" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/n8n-fresh-benchmark.yml ++++ b/.github/workflows/n8n-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 6" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "17,47 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-obs-studio.patch b/migration/cadence-cutover-patches/benchmark-obs-studio.patch new file mode 100644 index 000000000..cc403fc6b --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-obs-studio.patch @@ -0,0 +1,12 @@ +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "18,48 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch b/migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch new file mode 100644 index 000000000..edb4bcaa3 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-opentelemetry-java.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "20 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/opentelemetry-java-gradle-fresh-benchmark.yml ++++ b/.github/workflows/opentelemetry-java-gradle-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 4" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "20,50 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-posthog.patch b/migration/cadence-cutover-patches/benchmark-posthog.patch new file mode 100644 index 000000000..1d39967e1 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-posthog.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "21 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/posthog-fresh-benchmark.yml ++++ b/.github/workflows/posthog-fresh-benchmark.yml +@@ -25,8 +25,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 3 * * 3" + pull_request: + paths: + - ".boringcache.toml" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "21,51 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-qdrant.patch b/migration/cadence-cutover-patches/benchmark-qdrant.patch new file mode 100644 index 000000000..dcf34c5be --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-qdrant.patch @@ -0,0 +1,33 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "23 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/qdrant-fresh-benchmark.yml ++++ b/.github/workflows/qdrant-fresh-benchmark.yml +@@ -8,7 +8,6 @@ + cli_version: {description: "Optional exact BoringCache CLI release tag.", required: false, type: string, default: ""} + benchmark_id_suffix: {description: "Optional suffix for isolated comparison runs.", required: false, type: string, default: ""} + runner_label: {description: "Optional runner label for a manual main-branch experiment.", required: false, type: string, default: ""} +- schedule: [{cron: "0 3 * * 1"}] + pull_request: + paths: [".boringcache.toml", ".gitmodules", ".github/actions/qdrant-docker-benchmark/**", ".github/workflows/qdrant-benchmark.yml", ".github/workflows/qdrant-fresh-benchmark.yml", "scripts/**", "upstream"] + concurrency: {group: "${{ github.workflow }}-${{ github.ref_name }}", cancel-in-progress: false} +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "23,53 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-spring-ai.patch b/migration/cadence-cutover-patches/benchmark-spring-ai.patch new file mode 100644 index 000000000..11b4b3825 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-spring-ai.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "25 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/spring-ai-maven-fresh-benchmark.yml ++++ b/.github/workflows/spring-ai-maven-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 4 * * 4" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "25,55 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-storybook.patch b/migration/cadence-cutover-patches/benchmark-storybook.patch new file mode 100644 index 000000000..96b6efbb1 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-storybook.patch @@ -0,0 +1,34 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "27 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/storybook-fresh-benchmark.yml ++++ b/.github/workflows/storybook-fresh-benchmark.yml +@@ -15,8 +15,6 @@ + required: false + type: string + default: "" +- schedule: +- - cron: "0 4 * * 3" + pull_request: + paths: + - ".gitmodules" +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "26,56 * * * *" +- + permissions: + actions: write + contents: write diff --git a/migration/cadence-cutover-patches/benchmark-zed.patch b/migration/cadence-cutover-patches/benchmark-zed.patch new file mode 100644 index 000000000..a671118c6 --- /dev/null +++ b/migration/cadence-cutover-patches/benchmark-zed.patch @@ -0,0 +1,23 @@ +--- a/.github/workflows/canary.yml ++++ b/.github/workflows/canary.yml +@@ -1,8 +1,6 @@ + name: CLI Canary + + on: +- schedule: +- - cron: "29 2 * * *" + workflow_dispatch: + + permissions: +--- a/.github/workflows/sync.yml ++++ b/.github/workflows/sync.yml +@@ -2,9 +2,6 @@ + + on: + workflow_dispatch: +- schedule: +- - cron: "28,58 * * * *" +- + permissions: + actions: write + checks: read diff --git a/migration/cadence-schedule-inventory-2026-10-05.json b/migration/cadence-schedule-inventory-2026-10-05.json new file mode 100644 index 000000000..841964a24 --- /dev/null +++ b/migration/cadence-schedule-inventory-2026-10-05.json @@ -0,0 +1,765 @@ +{ + "schema_version": 1, + "observed_on": "2026-10-05", + "state": "observed; historical schedules unchanged", + "repositories": [ + { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "head_sha": "256f12b07ef648695134915651c1e03cf58cfd27", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "0e6e09dec41131d244295366fee4439f7b5673ff", + "name": "CLI Canary", + "schedule": [ + { + "cron": "2 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/chroma-fresh-benchmark.yml", + "blob_sha": "d6cfb307098fa27d68a35b61ab4eeb47a7147388", + "name": "Chroma Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "94bb5ec56874b55fb3d8cc68c134a58e0aff5636", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "1,31 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-deno", + "branch": "main", + "head_sha": "bc8769a7f3eac04392354e710f39b322323a82f0", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "eacd2e8581720d6c3d94f40d259d9bb1676dac43", + "name": "CLI Canary", + "schedule": [ + { + "cron": "3 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "e7b2672f5154fce0ee7a8d219a7680c1d2cf21c7", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "2,32 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "head_sha": "12a5323f222b7faf2c30bad5c21f4441dc233fb4", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "b045a246fd6123162d847d1fd2a2618991bcedf6", + "name": "CLI Canary", + "schedule": [ + { + "cron": "5 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/duckgres-fresh-benchmark.yml", + "blob_sha": "a746c482d1e9ea6cdd5a4e62122fa97e886efd8d", + "name": "Duckgres Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "f59b3a6fd1d3f5e70c202cdacedfe4c3ddd97a30", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "5,35 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "head_sha": "b135b70ad1349cd7993f53fe70dd01353885772d", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "72561e3a4d68432dcc9ee05ab311742580c60c9f", + "name": "CLI Canary", + "schedule": [ + { + "cron": "7 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/grpc-bazel-fresh-benchmark.yml", + "blob_sha": "e6a91c9fa2ed76a9cefbe89090d9fc88d30e57a3", + "name": "gRPC Bazel Fresh", + "schedule": [ + { + "cron": "0 3 * * 5" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "06633f53ceb3b98a58cc5a5f4dff859fd1d9598b", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "7,37 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "head_sha": "ecc41a3b3e250f5268000ddde142d61a15959e95", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "3437d0f6324e54b47df5d438bd6cfa6e96731ba8", + "name": "CLI Canary", + "schedule": [ + { + "cron": "9 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/hugo-fresh-benchmark.yml", + "blob_sha": "669a6acb6dd9aedb32a690c431bc60878be63c49", + "name": "Hugo Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "b18761d26ae89ee4ba65aabf1519104f0d0baf59", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "9,39 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "head_sha": "d6186956df7f400627bb2566bd907431801b191c", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "fc672d87343291eba494deaf13219a55df60e282", + "name": "CLI Canary", + "schedule": [ + { + "cron": "11 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/hugo-go-fresh-benchmark.yml", + "blob_sha": "a0b5eded84c1dea44b276c265ccb66d6f88ab50d", + "name": "Hugo Go Fresh", + "schedule": [ + { + "cron": "0 4 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "a9afa0c852ccb9ea423715f5448c6327db6ed2dc", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "10,40 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "head_sha": "46108490e1df3ac6edbe7e75fe220b25aab155eb", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "e1fbebad9cd370fdb2abcb341fd64debd4d5b979", + "name": "CLI Canary", + "schedule": [ + { + "cron": "12 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/immich-fresh-benchmark.yml", + "blob_sha": "b79f86d331337004a1288fdb6f88cb9387ab26d7", + "name": "Immich Fresh", + "schedule": [ + { + "cron": "0 3 * * 2" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "91bc18a30a993893f2cff0814995dee6dbce34c9", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "12,42 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "head_sha": "574b7c5b2a7f138fa2cba78dbfb1a33f3448f89b", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "4843be080f498736d6b00f4f63d602aa9df16053", + "name": "CLI Canary", + "schedule": [ + { + "cron": "14 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/linkerd2-fresh-benchmark.yml", + "blob_sha": "9520528d53ec96f56024f8fb54783e16d348eab8", + "name": "Linkerd2 Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "f6ccdc84f2b66c67163afd9d6f3cbc5069018852", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "13,43 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "head_sha": "2a9f1c12c8099b3c4094028bcdc9801237924fd0", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "26016bac6319ff72f486e3ef56f3493a486a73ec", + "name": "CLI Canary", + "schedule": [ + { + "cron": "16 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/mastodon-fresh-benchmark.yml", + "blob_sha": "1add987ea7b0c126182287a7476c805547e58d4c", + "name": "Mastodon Docker Fresh", + "schedule": [ + { + "cron": "0 4 * * 2" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "3fcf7fcac070ae5b1d87ce845f940eb37b8b5b99", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "15,45 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "head_sha": "52a6cb8d9acc3d569a8c156478242b85ffd2ff19", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "864b20860a0caf8c3038c392e38458dc536f2b8f", + "name": "CLI Canary", + "schedule": [ + { + "cron": "18 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/n8n-docker-fresh-benchmark.yml", + "blob_sha": "7fd5a970403e29deeae2acac3bb12587c32c1e57", + "name": "n8n Docker Fresh", + "schedule": [ + { + "cron": "30 3 * * 6" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/n8n-fresh-benchmark.yml", + "blob_sha": "774595c682017712b55deb7f20e02f60d232687a", + "name": "n8n Turbo Fresh", + "schedule": [ + { + "cron": "0 3 * * 6" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "d53eb06298a1f306694c00820e71680a3c26f34b", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "17,47 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "head_sha": "42d22b7f50da7e225effcb9ec9a2aa97c47f97cb", + "scheduled_workflows": [ + { + "path": ".github/workflows/sync.yml", + "blob_sha": "c81c212fa2d9b2e8cc5591ae41e5bc9cd7dfdaa9", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "18,48 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-opentelemetry-java", + "branch": "main", + "head_sha": "b83cdf213f54cc9572f122d0dfbf40087e1d15fd", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "496246c8fc77d89c7e1e419c1ff24afb1e57db32", + "name": "CLI Canary", + "schedule": [ + { + "cron": "20 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/opentelemetry-java-gradle-fresh-benchmark.yml", + "blob_sha": "d90e47f33890233759446efb76e3139c068a4c62", + "name": "OpenTelemetry Java Gradle Fresh", + "schedule": [ + { + "cron": "0 3 * * 4" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "fed2ce18754929d40cc75c0fcb731d56e05e97ae", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "20,50 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "head_sha": "0462819802f2b8a6393ee8221e2bc595f8b8f058", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "b5a974165b5eb4ee8b2d721c3da61096f43654c0", + "name": "CLI Canary", + "schedule": [ + { + "cron": "21 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/posthog-fresh-benchmark.yml", + "blob_sha": "64da0626ed0435e003d25afc524eeafee574d67c", + "name": "PostHog Fresh", + "schedule": [ + { + "cron": "0 3 * * 3" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "862f73e4e5bfe01ba86dc16886fe46be6eb3074f", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "21,51 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-qdrant", + "branch": "main", + "head_sha": "6dc192c834f1ac641969dbc49a0f352369e05a18", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "886ad9d34bd80bb81c606501ec8dc422e9a5d0d8", + "name": "CLI Canary", + "schedule": [ + { + "cron": "23 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/qdrant-fresh-benchmark.yml", + "blob_sha": "60e5e575d08b285886f782764b808c4059042d3c", + "name": "Qdrant Docker Fresh", + "schedule": [ + { + "cron": "0 3 * * 1" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "c77b7926b59082e33f988d30a42ff49ec3efc866", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "23,53 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-spring-ai", + "branch": "main", + "head_sha": "a7c6468d2f7c4d7d6f96692e52875c7b4c6b499b", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "3746cb6f27d3de5c90cc684005edde78c14d6b26", + "name": "CLI Canary", + "schedule": [ + { + "cron": "25 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/spring-ai-maven-fresh-benchmark.yml", + "blob_sha": "de595ad12aa86d8dd7dce8764fca9f9779f14ff2", + "name": "Spring AI Maven Fresh", + "schedule": [ + { + "cron": "0 4 * * 4" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "34054e8aa5851c8b5eb594d5e469b5de25d5de2e", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "25,55 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "head_sha": "681058e746f602b8d0717536fa5ccfee6a2330f7", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "736511aa88aa5dd42c64215491c7e02b646705f8", + "name": "CLI Canary", + "schedule": [ + { + "cron": "27 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/storybook-fresh-benchmark.yml", + "blob_sha": "b3c7bc31db4c67ede4023204720a5474411751c6", + "name": "Storybook Nx Fresh", + "schedule": [ + { + "cron": "0 4 * * 3" + } + ], + "other_events": [ + "workflow_dispatch", + "pull_request" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "52555528e53ffcbc6542f2f48014455837d3aac4", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "26,56 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + }, + { + "repository": "boringcache/benchmark-zed", + "branch": "main", + "head_sha": "a492769ef83da59ab47aa6a703910d64fb4d9eba", + "scheduled_workflows": [ + { + "path": ".github/workflows/canary.yml", + "blob_sha": "3a131e6f58d054993ab078a58a9a57c549169910", + "name": "CLI Canary", + "schedule": [ + { + "cron": "29 2 * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + }, + { + "path": ".github/workflows/sync.yml", + "blob_sha": "2c888b5a5cb32e6cdecf727bdb38be30bbc91851", + "name": "Sync Upstream", + "schedule": [ + { + "cron": "28,58 * * * *" + } + ], + "other_events": [ + "workflow_dispatch" + ] + } + ] + } + ] +} From 4cc237d1dbe67355979b0ec38ff39d646d1c7512 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:36:14 +0100 Subject: [PATCH 05/12] Keep unavailable OBS cache-hit evidence unmeasured --- .../payload/scripts/write_phase_result.rb | 1 + test/benchmark_case_scripts_test.rb | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/cases/obs-studio/payload/scripts/write_phase_result.rb b/cases/obs-studio/payload/scripts/write_phase_result.rb index 5ef340509..c6b99ae15 100755 --- a/cases/obs-studio/payload/scripts/write_phase_result.rb +++ b/cases/obs-studio/payload/scripts/write_phase_result.rb @@ -15,6 +15,7 @@ raise "Phase must be base or rolling" unless %w[base rolling].include?(phase) raise "Continuous observations require the rolling phase" if args["continuous"] && phase != "rolling" cache_hit = args["continuous"] ? args["cache_hit"] : (phase == "base" ? "false" : "true") +cache_hit = nil if cache_hit == "" raise "Cache hit must be true or false" if cache_hit && !%w[true false].include?(cache_hit) build_seconds = Float(args.fetch("build_seconds")) raise "Build time must be finite and nonnegative" unless build_seconds.finite? && build_seconds >= 0 diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 3e37df527..4636e6258 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -318,6 +318,21 @@ def test_reviewed_recipe_rejects_changed_upstream_files end end + def test_continuous_obs_report_keeps_an_absent_cache_hit_unmeasured + with_case("obs-studio") do |directory| + FileUtils.cp(File.join(ROOT, "scripts/canonical/benchmark-report.rb"), File.join(directory, "scripts/benchmark-report.rb")) + _, error, status = run_script(directory, "write_phase_result", "--surface", "xcode", + "--strategy", "boringcache", "--phase", "rolling", "--continuous", "--cache-hit", "", + "--source-sha", "a" * 40, "--restore-seconds", "0", "--build-seconds", "3") + assert status.success?, error + native = JSON.parse(File.read(File.join(directory, "benchmark-results/xcode-boringcache-rolling.json"))) + assert_equal "unmeasured", native.dig("classification", "cache_import_status") + record = JSON.parse(File.read(File.join(directory, "benchmark-results/obs-studio-boringcache-xcode-rolling-commit.json"))) + assert_nil record.dig("cache", "hit") + assert_equal 3.0, record.dig("timing", "build_seconds") + end + end + def test_selected_recipe_contract_checks_each_declared_plan with_case("zed") do |directory| source = File.join(directory, "upstream") From c99743723b7c2572ce86a4d66c5a2bf1e977a083 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:47:35 +0100 Subject: [PATCH 06/12] Retry verified concurrent source publication conflicts --- docs/cadence.md | 5 ++- .../mastodon-publication/commits.json | 1 + .../concurrency-after-fix.json | 26 +++++++++++ .../concurrency-before-fix.json | 26 +++++++++++ .../rehearsals/mastodon-publication/plan.md | 18 ++++++++ .../mastodon-publication/proposal.json | 14 ++++++ .../mastodon-publication/receipt.json | 44 +++++++++++++++++++ .../rehearsals/mastodon-publication/review.md | 29 ++++++++++++ scripts/source-promotion.rb | 10 ++++- test/source_promotion_test.rb | 31 ++++++++++++- 10 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 migration/rehearsals/mastodon-publication/commits.json create mode 100644 migration/rehearsals/mastodon-publication/concurrency-after-fix.json create mode 100644 migration/rehearsals/mastodon-publication/concurrency-before-fix.json create mode 100644 migration/rehearsals/mastodon-publication/plan.md create mode 100644 migration/rehearsals/mastodon-publication/proposal.json create mode 100644 migration/rehearsals/mastodon-publication/receipt.json create mode 100644 migration/rehearsals/mastodon-publication/review.md diff --git a/docs/cadence.md b/docs/cadence.md index bec67f2bf..0f85808bd 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -102,7 +102,10 @@ Remaining activation requirements: - Qualify the new and changed rolling paths with actual source advancement, retained seeds, verified outputs and canary CLI execution. - Pass the corrected Zed Nix cold/warm screen using the common dependency seed. -- Verify publication and receipt reconciliation in a bounded rehearsal. +- Verify main-branch workflow-token publication at cutover. The + [isolated publication rehearsal](../migration/rehearsals/mastodon-publication/review.md) + passed conditional commits, dispatch and reconciliation with the operator's + GitHub CLI; the hosted rehearsal passed inspection and dry-run publication. - Retire replaced historical cron triggers while preserving manual and release entrypoints; verify no outstanding duplicate dispatches. - After a compatible stable release, change monitoring and publication ownership diff --git a/migration/rehearsals/mastodon-publication/commits.json b/migration/rehearsals/mastodon-publication/commits.json new file mode 100644 index 000000000..94fe3286a --- /dev/null +++ b/migration/rehearsals/mastodon-publication/commits.json @@ -0,0 +1 @@ +[{"message":"Record mastodon rolling completed","sha":"540010c2ef47f38ecd2556b53830a27f053dc7fd","verified":true},{"message":"Record mastodon rolling requested","sha":"44b9954102b65bfc370bd78c6e8123ae999fa2a9","verified":true},{"message":"Record mastodon rolling dispatching","sha":"914e37913a240929d68908bb0e5b46db3c898ad9","verified":true}] diff --git a/migration/rehearsals/mastodon-publication/concurrency-after-fix.json b/migration/rehearsals/mastodon-publication/concurrency-after-fix.json new file mode 100644 index 000000000..36b8129c0 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/concurrency-after-fix.json @@ -0,0 +1,26 @@ +[ + { + "writer": "a", + "commit": "dc13d9d6712e088c7274d3bbd530e617c7989ebb", + "conflicts": [] + }, + { + "writer": "b", + "commit": "d06fa3645dc73f0ced3f296d38b757e91cd424a4", + "conflicts": [ + { + "type": "FORBIDDEN", + "path": [ + "createCommitOnBranch" + ], + "locations": [ + { + "line": 1, + "column": 48 + } + ], + "message": "is at dc13d9d6712e088c7274d3bbd530e617c7989ebb but expected 332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff" + } + ] + } +] diff --git a/migration/rehearsals/mastodon-publication/concurrency-before-fix.json b/migration/rehearsals/mastodon-publication/concurrency-before-fix.json new file mode 100644 index 000000000..18b6a3d21 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/concurrency-before-fix.json @@ -0,0 +1,26 @@ +[ + { + "writer": "a", + "error": "Source publication failed: is at 332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff but expected dbb5e9401a676589cfeabef936bcf08436bb0e10", + "conflicts": [ + { + "type": "FORBIDDEN", + "path": [ + "createCommitOnBranch" + ], + "locations": [ + { + "line": 1, + "column": 48 + } + ], + "message": "is at 332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff but expected dbb5e9401a676589cfeabef936bcf08436bb0e10" + } + ] + }, + { + "writer": "b", + "commit": "332d1ef92f7a1fd59d1a11f33a9f7f85cc0858ff", + "conflicts": [] + } +] diff --git a/migration/rehearsals/mastodon-publication/plan.md b/migration/rehearsals/mastodon-publication/plan.md new file mode 100644 index 000000000..92ac79b54 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/plan.md @@ -0,0 +1,18 @@ +# Source publication rehearsal + +Question: does the source publisher conditionally advance an isolated branch, +retain intent before dispatch, request both declared Mastodon rolling variants +with the exact canary, and reconcile their returned run IDs without modifying +main? + +The input is the Mastodon proposal verified by source-check run 37321566845. +Use branch cadence-rehearsal-publication and CLI vcli-canary-7a5b27146ebe. +Request server and streaming once each, with their declared GitHub Actions and +BoringCache providers. Use the branch's default rolling cache cohort. These are +bootstrap observations, not evidence of changed-source cache reuse. The separate +cadence-rolling seed and advancement series test cache reuse. + +Verify the published file diff, commit verification, exact returned run IDs, +retained intent and final receipts, reconciliation outcome, and unchanged main +head and activation setting. Retain failures and uncertain requests. Do not retry +an uncertain dispatch automatically. diff --git a/migration/rehearsals/mastodon-publication/proposal.json b/migration/rehearsals/mastodon-publication/proposal.json new file mode 100644 index 000000000..6013082b3 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/proposal.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "records": [ + { + "case_id": "mastodon", + "updated": true, + "base_sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "head_sha": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "state": "proposal", + "requires_case_qualification": true + } + ], + "publication": "requires-review" +} diff --git a/migration/rehearsals/mastodon-publication/receipt.json b/migration/rehearsals/mastodon-publication/receipt.json new file mode 100644 index 000000000..54c9264e0 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/receipt.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "state": "completed", + "cli_version": "vcli-canary-7a5b27146ebe", + "ref": "cadence-rehearsal-publication", + "proposal": { + "case_id": "mastodon", + "updated": true, + "base_sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "head_sha": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "state": "proposal", + "requires_case_qualification": true + }, + "created_at": "2026-10-05T14:13:24Z", + "runs": [ + { + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "variant": "server" + }, + "state": "requested", + "id": 37322979117, + "url": "https://github.com/boringcache/benchmarks/actions/runs/37322979117", + "outcome": "success" + }, + { + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "variant": "streaming" + }, + "state": "requested", + "id": 37322983836, + "url": "https://github.com/boringcache/benchmarks/actions/runs/37322983836", + "outcome": "success" + } + ] +} diff --git a/migration/rehearsals/mastodon-publication/review.md b/migration/rehearsals/mastodon-publication/review.md new file mode 100644 index 000000000..032ee3c30 --- /dev/null +++ b/migration/rehearsals/mastodon-publication/review.md @@ -0,0 +1,29 @@ +# Source publication rehearsal result + +The publisher advanced only `cadence-rehearsal-publication`. It committed intent +before making requests, retained both returned run IDs, then reconciled both +successful outcomes to `completed`. All three publication commits are verified +by GitHub. The changed paths were Mastodon's case definition and its current and +per-source rolling receipts. + +- Server: [run 37322979117](https://github.com/boringcache/benchmarks/actions/runs/37322979117). +- Streaming: [run 37322983836](https://github.com/boringcache/benchmarks/actions/runs/37322983836). +- Final receipt: [commit 540010c2](https://github.com/boringcache/benchmarks/commit/540010c2). + +The main branch remained at `b3855db2e9c0b27636cc143a772fe9917caf4940` throughout +this rehearsal, and `BENCHMARK_CADENCE_ACTIVE` remained unset. These are bootstrap +runs and do not establish changed-source cache reuse or comparative performance. + +The local rehearsal used the operator's authenticated GitHub CLI. The separate +hosted source rehearsal exercised the read-only inspector and dry-run publisher +with the workflow token; it did not exercise main-branch write permissions. +The first activated source cycle must verify those writes and its retained +receipts. No production activation is implied by this rehearsal. + +A concurrent-write check forced two publishers to read the same branch head. +The first check exposed a ref-lock race reported by GitHub as `FORBIDDEN` with +exact current and expected commit SHAs. The publisher now retries that specific +conflict as well as `STALE_DATA`, while rejecting ordinary permission failures. +The repeated check committed both independent files; one publisher encountered +and recovered from that exact conflict. The before/after responses remain beside +this review. These checks modified only the isolated rehearsal branch. diff --git a/scripts/source-promotion.rb b/scripts/source-promotion.rb index 1c5eb0b01..05b729d26 100644 --- a/scripts/source-promotion.rb +++ b/scripts/source-promotion.rb @@ -122,7 +122,15 @@ def commit(changes, expected:, message:) errors = result.fetch("errors", []) # Only an optimistic-concurrency conflict is safe to retry. Other API # failures may have an uncertain outcome and need their receipt inspected. - raise Error, "Source publication failed: #{errors.map { |error| error['message'] }.join('; ')}" unless errors.any? { |error| error["type"] == "STALE_DATA" } + retryable = errors.any? && errors.all? do |error| + # A ref-lock race after GraphQL's initial check is reported as + # FORBIDDEN. Retry only its exact old/new SHA response, not permission + # failures or an uncertain request outcome. + race = error["message"].to_s.match(/\Ais at ([0-9a-f]{40}) but expected #{Regexp.escape(head)}\z/) + error["type"] == "STALE_DATA" || + (error["type"] == "FORBIDDEN" && error["path"] == ["createCommitOnBranch"] && race && race[1] != head) + end + raise Error, "Source publication failed: #{errors.map { |error| error['message'] }.join('; ')}" unless retryable end raise Error, "Source publication could not acquire the current branch head" end diff --git a/test/source_promotion_test.rb b/test/source_promotion_test.rb index bc18da236..67e99581b 100644 --- a/test/source_promotion_test.rb +++ b/test/source_promotion_test.rb @@ -160,13 +160,42 @@ def test_publication_rejects_a_concurrent_edit_instead_of_overwriting_it end def test_unrelated_branch_advancement_retries_with_a_new_expected_head + [ + {"type" => "STALE_DATA", "message" => "Branch advanced"}, + {"type" => "FORBIDDEN", "path" => ["createCommitOnBranch"], "message" => "is at #{'c' * 40} but expected #{'b' * 40}"} + ].each { |conflict| assert_concurrent_commit_retries(conflict) } + end + + def test_permission_failures_and_unrelated_ref_errors_are_not_retried + ["Resource not accessible by integration", "is at #{'c' * 40} but expected #{'a' * 40}"].each do |message| + publisher = SourcePromotion::Publisher.new + requests = 0 + api = lambda do |path, body: nil| + if path == "graphql" + requests += 1 + {"errors" => [{"type" => "FORBIDDEN", "path" => ["createCommitOnBranch"], "message" => message}]} + else + {"object" => {"sha" => "b" * 40}} + end + end + publisher.stub(:api, api) do + error = assert_raises(SourcePromotion::Error) { publisher.commit({"receipt.json" => "{}"}, expected: {}, message: "Record receipt") } + assert_includes error.message, message + end + assert_equal 1, requests + end + end + + private + + def assert_concurrent_commit_retries(conflict) publisher = SourcePromotion::Publisher.new heads = ["b" * 40, "c" * 40] commits = [] api = lambda do |path, body: nil| if path == "graphql" commits << body.fetch("variables").fetch("input") - commits.length == 1 ? {"errors" => [{"type" => "STALE_DATA", "message" => "Branch advanced"}]} : {"data" => {"createCommitOnBranch" => {"commit" => {"oid" => "d" * 40}}}} + commits.length == 1 ? {"errors" => [conflict]} : {"data" => {"createCommitOnBranch" => {"commit" => {"oid" => "d" * 40}}}} elsif path.include?("git/ref") {"object" => {"sha" => heads.shift}} else From 8ecc75a909607be0771c1b1aca266f1360060aad Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:52:04 +0100 Subject: [PATCH 07/12] Distinguish stable and canary benchmark runs consistently --- .github/workflows/deno-deno-cargo-product.yml | 3 +-- .github/workflows/deno-deno-cargo-rolling-chain.yml | 4 +--- .github/workflows/grpc-grpc-bazel-benchmark.yml | 4 +--- .github/workflows/grpc-grpc-bazel-fresh-benchmark.yml | 6 +----- .github/workflows/native-fresh-benchmark.yml | 3 +-- .github/workflows/native-rolling-benchmark.yml | 3 +-- .github/workflows/nix-fresh-benchmark.yml | 3 +-- .github/workflows/nix-rolling-benchmark.yml | 3 +-- .github/workflows/obs-rolling-benchmark.yml | 3 +-- .github/workflows/obs-studio-obs-actions-cache.yml | 5 +---- .github/workflows/obs-studio-obs-boringcache.yml | 6 +----- .github/workflows/reapi-fresh-benchmark.yml | 3 +-- .github/workflows/reapi-rolling-benchmark.yml | 3 +-- .github/workflows/zed-zed-cargo-product.yml | 2 +- .github/workflows/zed-zed-cargo-rolling-auto.yml | 3 +-- docs/cadence.md | 8 ++++++-- 16 files changed, 21 insertions(+), 41 deletions(-) diff --git a/.github/workflows/deno-deno-cargo-product.yml b/.github/workflows/deno-deno-cargo-product.yml index 4d9bf872a..d0e70ca4b 100644 --- a/.github/workflows/deno-deno-cargo-product.yml +++ b/.github/workflows/deno-deno-cargo-product.yml @@ -1,7 +1,6 @@ --- name: 'deno: Deno Cargo product run' -run-name: "${{ format('Deno Cargo fresh | {0} | CLI {1}', inputs.cache_profile || - 'cargo-product', inputs.cli_version || 'Action default') }}" +run-name: "Deno Cargo / ${{ inputs.cache_profile || 'cargo-product' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/deno-deno-cargo-rolling-chain.yml b/.github/workflows/deno-deno-cargo-rolling-chain.yml index 192cb0357..9197228ec 100644 --- a/.github/workflows/deno-deno-cargo-rolling-chain.yml +++ b/.github/workflows/deno-deno-cargo-rolling-chain.yml @@ -1,8 +1,6 @@ --- name: 'deno: Deno Cargo rolling run' -run-name: "${{ github.event_name == 'push' && 'Deno Cargo rolling pinned source update' - || format('Deno Cargo rolling {0} | {1} to {2}', inputs.cache_profile || 'cargo-product', - inputs.base_sha, inputs.head_sha) }}" +run-name: "Deno Cargo / ${{ inputs.cache_profile || 'cargo-product' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.base_sha }} to ${{ inputs.head_sha }}" permissions: actions: read contents: read diff --git a/.github/workflows/grpc-grpc-bazel-benchmark.yml b/.github/workflows/grpc-grpc-bazel-benchmark.yml index 0e30aa07d..c1b94da69 100644 --- a/.github/workflows/grpc-grpc-bazel-benchmark.yml +++ b/.github/workflows/grpc-grpc-bazel-benchmark.yml @@ -1,8 +1,6 @@ --- name: 'grpc: gRPC Bazel Benchmark' -run-name: "${{ format('Benchmark {0} | {1}', inputs.cli_version != '' && 'Canary' - || 'Rolling', github.event_name == 'push' && 'Upstream update' || format('Manual - dispatch ({0})', github.ref_name)) }}" +run-name: "gRPC Bazel | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml b/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml index 41ec801dc..12972f3a6 100644 --- a/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml +++ b/.github/workflows/grpc-grpc-bazel-fresh-benchmark.yml @@ -1,10 +1,6 @@ --- name: 'grpc: gRPC Bazel Fresh' -run-name: "${{ format('Benchmark {0} | {1}', github.event_name == 'pull_request' && - 'Validation' || inputs.cli_version != '' && 'Canary' || 'Fresh', github.event_name - == 'schedule' && 'Weekly' || github.event_name == 'pull_request' && format('PR #{0}', - github.event.pull_request.number) || format('Manual dispatch ({0})', github.ref_name)) - }}" +run-name: "gRPC Bazel | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/native-fresh-benchmark.yml b/.github/workflows/native-fresh-benchmark.yml index 0ba1421f2..6de0ed6d6 100644 --- a/.github/workflows/native-fresh-benchmark.yml +++ b/.github/workflows/native-fresh-benchmark.yml @@ -1,7 +1,6 @@ --- name: Native comparison fresh -run-name: "${{ inputs.case_id }} ${{ inputs.variant }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }}${{ inputs.variant != '' && format(' / {0}', inputs.variant) || '' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/native-rolling-benchmark.yml b/.github/workflows/native-rolling-benchmark.yml index 07eadad28..734ae269c 100644 --- a/.github/workflows/native-rolling-benchmark.yml +++ b/.github/workflows/native-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: Native comparison rolling -run-name: "${{ inputs.case_id }} ${{ inputs.variant }} rolling ${{ inputs.series_id - }} sample ${{ inputs.sample }}" +run-name: "${{ inputs.case_id }}${{ inputs.variant != '' && format(' / {0}', inputs.variant) || '' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/nix-fresh-benchmark.yml b/.github/workflows/nix-fresh-benchmark.yml index 0dacd26dc..ae94d13d8 100644 --- a/.github/workflows/nix-fresh-benchmark.yml +++ b/.github/workflows/nix-fresh-benchmark.yml @@ -1,7 +1,6 @@ --- name: Nix comparison fresh -run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/nix-rolling-benchmark.yml b/.github/workflows/nix-rolling-benchmark.yml index 59bce02d4..f684684cd 100644 --- a/.github/workflows/nix-rolling-benchmark.yml +++ b/.github/workflows/nix-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: Nix comparison rolling -run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/obs-rolling-benchmark.yml b/.github/workflows/obs-rolling-benchmark.yml index cb84978e9..32cf94945 100644 --- a/.github/workflows/obs-rolling-benchmark.yml +++ b/.github/workflows/obs-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: OBS Studio comparison rolling -run-name: obs-studio ${{ inputs.cache_tool }} rolling ${{ inputs.series_id }} sample - ${{ inputs.sample }} +run-name: "obs-studio / ${{ inputs.cache_tool || 'all' }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" 'on': workflow_dispatch: inputs: diff --git a/.github/workflows/obs-studio-obs-actions-cache.yml b/.github/workflows/obs-studio-obs-actions-cache.yml index f86ba64ca..2233dbccc 100644 --- a/.github/workflows/obs-studio-obs-actions-cache.yml +++ b/.github/workflows/obs-studio-obs-actions-cache.yml @@ -1,9 +1,6 @@ --- name: 'obs-studio: OBS Studio Actions Cache benchmark' -run-name: "${{ github.event_name == 'pull_request' && format('OBS validation | PR - #{0}', github.event.pull_request.number) || github.event_name == 'push' && 'OBS - rolling | Upstream update' || format('OBS Actions Cache | Manual dispatch ({0})', - github.ref_name) }}" +run-name: "obs-studio / ${{ inputs.cache_tool || 'all' }} / Actions Cache | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/obs-studio-obs-boringcache.yml b/.github/workflows/obs-studio-obs-boringcache.yml index 7796b7f01..423384c5a 100644 --- a/.github/workflows/obs-studio-obs-boringcache.yml +++ b/.github/workflows/obs-studio-obs-boringcache.yml @@ -1,10 +1,6 @@ --- name: 'obs-studio: OBS Studio BoringCache benchmark' -run-name: "${{ github.event_name == 'pull_request' && format('OBS validation | PR - #{0}', github.event.pull_request.number) || inputs.cli_version != '' && format('OBS - canary | CLI {0}', inputs.cli_version) || github.event_name == 'push' && 'OBS rolling - | Upstream update' || format('OBS BoringCache | Manual dispatch ({0})', github.ref_name) - }}" +run-name: "obs-studio / ${{ inputs.cache_tool || 'all' }} / BoringCache | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/reapi-fresh-benchmark.yml b/.github/workflows/reapi-fresh-benchmark.yml index b1b322b74..3ffa84c1c 100644 --- a/.github/workflows/reapi-fresh-benchmark.yml +++ b/.github/workflows/reapi-fresh-benchmark.yml @@ -1,7 +1,6 @@ --- name: REAPI comparison fresh -run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/reapi-rolling-benchmark.yml b/.github/workflows/reapi-rolling-benchmark.yml index 178ca1d4a..11149202d 100644 --- a/.github/workflows/reapi-rolling-benchmark.yml +++ b/.github/workflows/reapi-rolling-benchmark.yml @@ -1,7 +1,6 @@ --- name: REAPI comparison rolling -run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: contents: read actions: read diff --git a/.github/workflows/zed-zed-cargo-product.yml b/.github/workflows/zed-zed-cargo-product.yml index 8682a5cd0..117059ecb 100644 --- a/.github/workflows/zed-zed-cargo-product.yml +++ b/.github/workflows/zed-zed-cargo-product.yml @@ -1,6 +1,6 @@ --- name: 'zed: Zed Cargo layer matrix' -run-name: "${{ format('Zed Cargo layers | CLI {0}', inputs.cli_version || 'Action default') }}" +run-name: "Zed Cargo layers | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} fresh | ${{ inputs.series_id || github.ref_name }} / sample ${{ inputs.sample || '1' }}" permissions: actions: read contents: read diff --git a/.github/workflows/zed-zed-cargo-rolling-auto.yml b/.github/workflows/zed-zed-cargo-rolling-auto.yml index 216471156..72e1115b6 100644 --- a/.github/workflows/zed-zed-cargo-rolling-auto.yml +++ b/.github/workflows/zed-zed-cargo-rolling-auto.yml @@ -1,7 +1,6 @@ --- name: 'zed: Zed Cargo rolling run' -run-name: "${{ format('Zed Cargo rolling {0} to {1}', inputs.base_sha, inputs.head_sha) - }}" +run-name: "Zed Cargo | ${{ startsWith(inputs.cli_version, 'vcli-canary-') && 'Canary' || 'Stable' }} rolling | ${{ inputs.base_sha }} to ${{ inputs.head_sha }}" permissions: actions: read checks: read diff --git a/docs/cadence.md b/docs/cadence.md index 0f85808bd..188d94ac4 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -73,8 +73,12 @@ keeping strict package and dependency comparisons. The earlier Zed Nix screen restored both providers' package outputs correctly but failed the dependency baseline comparison; that failed run remains retained. -Shared workflows use case, variant, lane, provider and phase labels. REAPI, Nix -and OBS rolling results use the canonical benchmark report and retain both +Run names use the case, optional variant, CLI channel and lane, followed by the +series or branch and sample. For example, `n8n / turbo | Canary fresh | main / +sample 1` and `n8n / turbo | Stable rolling | main / sample 1`. Stable tags are +not labelled as canaries merely because an exact CLI version was supplied. +Provider and phase remain visible in job labels. REAPI, Nix and OBS rolling +results use the canonical benchmark report and retain both structured evidence and the comparison summary. Reports contain measurements and verification results, without performance verdicts. From a9879f17864ecb4a7abe299a1492f0ad4be50d1d Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:54:11 +0100 Subject: [PATCH 08/12] Collect canonical observations from mixed phase bundles --- scripts/collect-series.rb | 7 ++++++- test/benchmark_series_test.rb | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/collect-series.rb b/scripts/collect-series.rb index 03812f4f3..3c09999e5 100644 --- a/scripts/collect-series.rb +++ b/scripts/collect-series.rb @@ -26,8 +26,13 @@ def self.call(directory, evidence_directory:, sample:) names.lines.map(&:strip).grep(/\.json\z/).each do |name| # Proof bundles also contain raw product evidence. Only collect canonical # phase filenames or the contents of a dedicated phase artifact. - next unless artifact.fetch("name").start_with?("phase-") || name.match?(/-(?:fresh-(?:cold|warm)|rolling-commit)\.json\z/) + canonical_name = name.match?(/-(?:fresh-(?:cold|warm)|rolling-commit)\.json\z/) + next unless artifact.fetch("name").start_with?("phase-") || canonical_name value = JSON.parse(read_record(archive, name)) + # Some phase bundles also retain native counters and provider receipts. + # Keep those in the export without treating them as series observations. + # A canonical filename still requires the complete record contract. + next unless canonical_name || (value.is_a?(Hash) && value.key?("series")) BenchmarkSeries.validate_record(plan, value) unless value.dig("github", "run_id").to_s == run_id && value.dig("series", "sample") == sample raise BenchmarkSeries::Error, "Phase artifact belongs to another run or sample" diff --git a/test/benchmark_series_test.rb b/test/benchmark_series_test.rb index d01cc86a8..8bae32d61 100644 --- a/test/benchmark_series_test.rb +++ b/test/benchmark_series_test.rb @@ -208,6 +208,24 @@ def test_proof_bundle_records_still_require_matching_case_and_dispatch assert_empty Dir[File.join(@directory, "runs", "*.json")] end + def test_phase_bundles_can_retain_native_counters_alongside_the_canonical_record + value = record + directory = phase_export([value], artifact_name: "phase-example-boringcache-release-rolling-commit", + filenames: ["example-boringcache-release-rolling-commit.json"], + extra_files: {"ccache-native.json" => {"direct_cache_hit" => 8}, + "ccache-rolling.json" => {"schema_version" => 2, "timing" => {"build_seconds" => 3}}}) + result = CollectSeries.call(@directory, evidence_directory: directory, sample: 1) + assert_equal 1, result["records"] + assert_equal value, JSON.parse(File.read(File.join(@directory, "runs", "1-boringcache-commit.json"))) + end + + def test_canonical_phase_filename_cannot_hide_a_missing_series_contract + directory = phase_export([record.except("series")], + filenames: ["example-boringcache-release-rolling-commit.json"]) + assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } + assert_empty Dir[File.join(@directory, "runs", "*.json")] + end + def test_collection_requires_the_original_dispatch_and_matching_run directory = phase_export([record.merge("github" => {"run_id" => "2"})]) assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } From 6ecea89df9e4b7427cac1eec329c161df27f19bc Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 16:22:47 +0100 Subject: [PATCH 09/12] Retain cadence qualification results and verified archives --- docs/cadence-qualification-2026-10-05.md | 61 ++ docs/cadence.md | 10 +- .../cadence-qualification-2026-10-05.json | 936 ++++++++++++++++++ migration/evidence.json | 756 ++++++++++++++ .../completions/37324204556.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-advance-02/report.json | 350 +++++++ .../cadence-rolling-advance-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 147 +++ .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-advance-02/series.json | 49 + .../completions/37323107836.json | 18 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-advance/report.json | 71 ++ .../cadence-rolling-advance/report.md | 38 + .../cadence-rolling-advance/series.json | 49 + .../completions/37324032100.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-seed-02/report.json | 350 +++++++ .../cadence-rolling-seed-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 147 +++ .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-seed-02/series.json | 49 + .../completions/37322592957.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-seed/report.json | 350 +++++++ .../cadence-rolling-seed/report.md | 30 + .../runs/1-bazel-remote-commit.json | 147 +++ .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-seed/series.json | 49 + .../completions/37324209639.json | 17 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance-02/report.json | 220 ++++ .../cadence-rolling-advance-02/report.md | 38 + .../runs/1-bazel-remote-commit.json | 146 +++ .../cadence-rolling-advance-02/series.json | 48 + .../completions/37325282767.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance-03/report.json | 348 +++++++ .../cadence-rolling-advance-03/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-advance-03/series.json | 48 + .../completions/37323112601.json | 17 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance/report.json | 222 +++++ .../cadence-rolling-advance/report.md | 38 + .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-advance/series.json | 48 + .../completions/37324016178.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed-02/report.json | 348 +++++++ .../cadence-rolling-seed-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed-02/series.json | 48 + .../completions/37325135541.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed-03/report.json | 348 +++++++ .../cadence-rolling-seed-03/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed-03/series.json | 48 + .../completions/37322598011.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed/report.json | 348 +++++++ .../gogs-moon/cadence-rolling-seed/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed/series.json | 48 + .../completions/37320867908.json | 13 + .../2-nix-fresh-benchmark-retry.yml.json | 37 + .../dispatches/2-nix-fresh-benchmark.yml.json | 36 + .../helix-nix/cadence-fresh-01/report.json | 801 +++++++++++++++ results/helix-nix/cadence-fresh-01/report.md | 36 + .../runs/1-boringcache-cold.json | 194 ++++ .../runs/1-boringcache-warm.json | 195 ++++ .../cadence-fresh-01/runs/1-cachix-cold.json | 152 +++ .../cadence-fresh-01/runs/1-cachix-warm.json | 152 +++ .../completions/37323117664.json | 13 + .../1-nix-rolling-benchmark.yml.json | 36 + .../cadence-rolling-advance/report.json | 412 ++++++++ .../cadence-rolling-advance/report.md | 29 + .../runs/1-boringcache-commit.json | 195 ++++ .../runs/1-cachix-commit.json | 152 +++ .../cadence-rolling-advance/series.json | 50 + .../completions/37322604229.json | 13 + .../1-nix-rolling-benchmark.yml.json | 36 + .../cadence-rolling-seed/report.json | 407 ++++++++ .../helix-nix/cadence-rolling-seed/report.md | 29 + .../runs/1-boringcache-commit.json | 194 ++++ .../runs/1-cachix-commit.json | 152 +++ .../cadence-rolling-seed/series.json | 50 + .../completions/37323121773.json | 13 + .../1-native-rolling-benchmark.yml.json | 34 + .../cadence-rolling-advance/report.json | 445 +++++++++ .../immich/cadence-rolling-advance/report.md | 29 + .../runs/1-actions-cache-commit.json | 167 ++++ .../runs/1-boringcache-commit.json | 213 ++++ .../cadence-rolling-advance/series.json | 45 + .../completions/37321739971.json | 13 + .../1-native-rolling-benchmark.yml.json | 34 + .../immich/cadence-rolling-seed/report.json | 445 +++++++++ results/immich/cadence-rolling-seed/report.md | 29 + .../runs/1-actions-cache-commit.json | 167 ++++ .../runs/1-boringcache-commit.json | 213 ++++ .../immich/cadence-rolling-seed/series.json | 45 + .../completions/37323126507.json | 13 + .../1-native-rolling-benchmark.yml.json | 39 + .../report.json | 525 ++++++++++ .../cadence-rolling-advance-server/report.md | 29 + .../runs/1-actions-cache-commit.json | 207 ++++ .../runs/1-boringcache-commit.json | 253 +++++ .../series.json | 46 + .../completions/37323131290.json | 13 + .../1-native-rolling-benchmark.yml.json | 39 + .../report.json | 525 ++++++++++ .../report.md | 29 + .../runs/1-actions-cache-commit.json | 207 ++++ .../runs/1-boringcache-commit.json | 253 +++++ .../series.json | 46 + .../completions/37321744434.json | 13 + .../1-native-rolling-benchmark.yml.json | 39 + .../cadence-rolling-seed-server/report.json | 525 ++++++++++ .../cadence-rolling-seed-server/report.md | 29 + .../runs/1-actions-cache-commit.json | 207 ++++ .../runs/1-boringcache-commit.json | 253 +++++ .../cadence-rolling-seed-server/series.json | 46 + .../completions/37321748690.json | 13 + .../1-native-rolling-benchmark.yml.json | 39 + .../report.json | 525 ++++++++++ .../cadence-rolling-seed-streaming/report.md | 29 + .../runs/1-actions-cache-commit.json | 207 ++++ .../runs/1-boringcache-commit.json | 253 +++++ .../series.json | 46 + .../completions/37324214890.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-advance-02/report.json | 350 +++++++ .../cadence-rolling-advance-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 147 +++ .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-advance-02/series.json | 49 + .../completions/37323136088.json | 17 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-advance/report.json | 223 +++++ .../cadence-rolling-advance/report.md | 38 + .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-advance/series.json | 49 + .../completions/37324036688.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-seed-02/report.json | 350 +++++++ .../cadence-rolling-seed-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 147 +++ .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-seed-02/series.json | 49 + .../completions/37322609279.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 36 + .../cadence-rolling-seed/report.json | 350 +++++++ .../cadence-rolling-seed/report.md | 30 + .../runs/1-bazel-remote-commit.json | 147 +++ .../runs/1-boringcache-commit.json | 149 +++ .../cadence-rolling-seed/series.json | 49 + .../completions/37323140444.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../report.json | 573 +++++++++++ .../report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 277 ++++++ .../series.json | 46 + .../completions/37323145174.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../report.json | 573 +++++++++++ .../cadence-rolling-advance-docker/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 277 ++++++ .../series.json | 46 + .../completions/37323150431.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../report.json | 573 +++++++++++ .../cadence-rolling-advance-runners/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 277 ++++++ .../series.json | 46 + .../completions/37323155106.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../cadence-rolling-advance-turbo/report.json | 564 +++++++++++ .../cadence-rolling-advance-turbo/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 268 +++++ .../cadence-rolling-advance-turbo/series.json | 46 + .../completions/37325183094.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../cadence-rolling-next-turbo/report.json | 564 +++++++++++ .../n8n/cadence-rolling-next-turbo/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 268 +++++ .../cadence-rolling-next-turbo/series.json | 46 + .../completions/37321753318.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../report.json | 573 +++++++++++ .../cadence-rolling-seed-distroless/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 277 ++++++ .../series.json | 46 + .../completions/37321758035.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../cadence-rolling-seed-docker/report.json | 573 +++++++++++ .../n8n/cadence-rolling-seed-docker/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 277 ++++++ .../cadence-rolling-seed-docker/series.json | 46 + .../completions/37321762921.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../cadence-rolling-seed-runners/report.json | 573 +++++++++++ .../cadence-rolling-seed-runners/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 277 ++++++ .../cadence-rolling-seed-runners/series.json | 46 + .../completions/37321767409.json | 13 + .../1-native-rolling-benchmark.yml.json | 40 + .../cadence-rolling-seed-turbo/report.json | 564 +++++++++++ .../n8n/cadence-rolling-seed-turbo/report.md | 29 + .../runs/1-actions-cache-commit.json | 231 +++++ .../runs/1-boringcache-commit.json | 268 +++++ .../cadence-rolling-seed-turbo/series.json | 46 + .../completions/37324219008.json | 13 + .../1-obs-rolling-benchmark.yml.json | 42 + .../report.json | 490 +++++++++ .../report.md | 29 + .../runs/1-actions-cache-commit.json | 197 ++++ .../runs/1-boringcache-commit.json | 224 +++++ .../series.json | 51 + .../completions/37326340221.json | 13 + .../1-obs-rolling-benchmark.yml.json | 42 + .../report.json | 490 +++++++++ .../report.md | 29 + .../runs/1-actions-cache-commit.json | 197 ++++ .../runs/1-boringcache-commit.json | 224 +++++ .../series.json | 51 + .../completions/37323234591.json | 17 + .../1-obs-rolling-benchmark.yml.json | 42 + .../cadence-rolling-advance-xcode/report.json | 278 ++++++ .../cadence-rolling-advance-xcode/report.md | 37 + .../runs/1-actions-cache-commit.json | 197 ++++ .../cadence-rolling-advance-xcode/series.json | 51 + .../completions/37323465858.json | 13 + .../1-obs-rolling-benchmark.yml.json | 42 + .../report.json | 470 +++++++++ .../cadence-rolling-seed-ccache-02/report.md | 29 + .../runs/1-actions-cache-commit.json | 181 ++++ .../runs/1-boringcache-commit.json | 224 +++++ .../series.json | 51 + .../completions/37322616359.json | 17 + .../1-obs-rolling-benchmark.yml.json | 42 + .../cadence-rolling-seed-ccache/report.json | 258 +++++ .../cadence-rolling-seed-ccache/report.md | 37 + .../runs/1-actions-cache-commit.json | 181 ++++ .../cadence-rolling-seed-ccache/series.json | 51 + .../completions/37326133177.json | 13 + .../1-obs-rolling-benchmark.yml.json | 42 + .../cadence-rolling-seed-xcode-02/report.json | 470 +++++++++ .../cadence-rolling-seed-xcode-02/report.md | 29 + .../runs/1-actions-cache-commit.json | 181 ++++ .../runs/1-boringcache-commit.json | 224 +++++ .../cadence-rolling-seed-xcode-02/series.json | 51 + .../completions/37322621030.json | 17 + .../1-obs-rolling-benchmark.yml.json | 42 + .../cadence-rolling-seed-xcode/report.json | 258 +++++ .../cadence-rolling-seed-xcode/report.md | 37 + .../runs/1-actions-cache-commit.json | 181 ++++ .../cadence-rolling-seed-xcode/series.json | 51 + .../completions/37324223748.json | 17 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance-02/report.json | 220 ++++ .../cadence-rolling-advance-02/report.md | 38 + .../runs/1-bazel-remote-commit.json | 146 +++ .../cadence-rolling-advance-02/series.json | 48 + .../completions/37325287254.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance-03/report.json | 348 +++++++ .../cadence-rolling-advance-03/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-advance-03/series.json | 48 + .../completions/37323159608.json | 17 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance/report.json | 222 +++++ .../cadence-rolling-advance/report.md | 38 + .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-advance/series.json | 48 + .../completions/37324020999.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed-02/report.json | 348 +++++++ .../cadence-rolling-seed-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed-02/series.json | 48 + .../completions/37325140807.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed-03/report.json | 348 +++++++ .../cadence-rolling-seed-03/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed-03/series.json | 48 + .../completions/37322625973.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed/report.json | 348 +++++++ .../cadence-rolling-seed/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed/series.json | 48 + .../1-native-rolling-benchmark.yml.json | 38 + .../series.json | 46 + .../completions/37321772493.json | 13 + .../1-native-rolling-benchmark.yml.json | 38 + .../cadence-rolling-seed-layers/report.json | 509 ++++++++++ .../cadence-rolling-seed-layers/report.md | 29 + .../runs/1-actions-cache-commit.json | 199 ++++ .../runs/1-boringcache-commit.json | 245 +++++ .../cadence-rolling-seed-layers/series.json | 46 + .../completions/37324228460.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance-02/report.json | 348 +++++++ .../cadence-rolling-advance-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-advance-02/series.json | 48 + .../completions/37323169828.json | 17 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-advance/report.json | 222 +++++ .../cadence-rolling-advance/report.md | 38 + .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-advance/series.json | 48 + .../completions/37324025714.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed-02/report.json | 348 +++++++ .../cadence-rolling-seed-02/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed-02/series.json | 48 + .../completions/37322630658.json | 13 + .../1-reapi-rolling-benchmark.yml.json | 35 + .../cadence-rolling-seed/report.json | 348 +++++++ .../cadence-rolling-seed/report.md | 30 + .../runs/1-bazel-remote-commit.json | 146 +++ .../runs/1-boringcache-commit.json | 148 +++ .../cadence-rolling-seed/series.json | 48 + .../dispatches/2-nix-fresh-benchmark.yml.json | 37 + .../1-nix-rolling-benchmark.yml.json | 36 + .../cadence-rolling-advance/series.json | 50 + .../1-nix-rolling-benchmark.yml.json | 36 + .../zed-nix/cadence-rolling-seed/series.json | 50 + scripts/benchmark-catalog.rb | 3 +- test/benchmark_catalog_test.rb | 2 + 354 files changed, 45940 insertions(+), 5 deletions(-) create mode 100644 docs/cadence-qualification-2026-10-05.md create mode 100644 migration/cadence-qualification-2026-10-05.json create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/report.json create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/report.md create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json create mode 100644 results/executorch-buck2/cadence-rolling-advance-02/series.json create mode 100644 results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json create mode 100644 results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/executorch-buck2/cadence-rolling-advance/report.json create mode 100644 results/executorch-buck2/cadence-rolling-advance/report.md create mode 100644 results/executorch-buck2/cadence-rolling-advance/series.json create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/report.json create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/report.md create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json create mode 100644 results/executorch-buck2/cadence-rolling-seed-02/series.json create mode 100644 results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json create mode 100644 results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/executorch-buck2/cadence-rolling-seed/report.json create mode 100644 results/executorch-buck2/cadence-rolling-seed/report.md create mode 100644 results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json create mode 100644 results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/executorch-buck2/cadence-rolling-seed/series.json create mode 100644 results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json create mode 100644 results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/gogs-moon/cadence-rolling-advance-02/report.json create mode 100644 results/gogs-moon/cadence-rolling-advance-02/report.md create mode 100644 results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json create mode 100644 results/gogs-moon/cadence-rolling-advance-02/series.json create mode 100644 results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json create mode 100644 results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/gogs-moon/cadence-rolling-advance-03/report.json create mode 100644 results/gogs-moon/cadence-rolling-advance-03/report.md create mode 100644 results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json create mode 100644 results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json create mode 100644 results/gogs-moon/cadence-rolling-advance-03/series.json create mode 100644 results/gogs-moon/cadence-rolling-advance/completions/37323112601.json create mode 100644 results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/gogs-moon/cadence-rolling-advance/report.json create mode 100644 results/gogs-moon/cadence-rolling-advance/report.md create mode 100644 results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/gogs-moon/cadence-rolling-advance/series.json create mode 100644 results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json create mode 100644 results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/gogs-moon/cadence-rolling-seed-02/report.json create mode 100644 results/gogs-moon/cadence-rolling-seed-02/report.md create mode 100644 results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json create mode 100644 results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json create mode 100644 results/gogs-moon/cadence-rolling-seed-02/series.json create mode 100644 results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json create mode 100644 results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/gogs-moon/cadence-rolling-seed-03/report.json create mode 100644 results/gogs-moon/cadence-rolling-seed-03/report.md create mode 100644 results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json create mode 100644 results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json create mode 100644 results/gogs-moon/cadence-rolling-seed-03/series.json create mode 100644 results/gogs-moon/cadence-rolling-seed/completions/37322598011.json create mode 100644 results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/gogs-moon/cadence-rolling-seed/report.json create mode 100644 results/gogs-moon/cadence-rolling-seed/report.md create mode 100644 results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json create mode 100644 results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/gogs-moon/cadence-rolling-seed/series.json create mode 100644 results/helix-nix/cadence-fresh-01/completions/37320867908.json create mode 100644 results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json create mode 100644 results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json create mode 100644 results/helix-nix/cadence-fresh-01/report.json create mode 100644 results/helix-nix/cadence-fresh-01/report.md create mode 100644 results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json create mode 100644 results/helix-nix/cadence-rolling-advance/completions/37323117664.json create mode 100644 results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json create mode 100644 results/helix-nix/cadence-rolling-advance/report.json create mode 100644 results/helix-nix/cadence-rolling-advance/report.md create mode 100644 results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json create mode 100644 results/helix-nix/cadence-rolling-advance/series.json create mode 100644 results/helix-nix/cadence-rolling-seed/completions/37322604229.json create mode 100644 results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json create mode 100644 results/helix-nix/cadence-rolling-seed/report.json create mode 100644 results/helix-nix/cadence-rolling-seed/report.md create mode 100644 results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json create mode 100644 results/helix-nix/cadence-rolling-seed/series.json create mode 100644 results/immich/cadence-rolling-advance/completions/37323121773.json create mode 100644 results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/immich/cadence-rolling-advance/report.json create mode 100644 results/immich/cadence-rolling-advance/report.md create mode 100644 results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json create mode 100644 results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/immich/cadence-rolling-advance/series.json create mode 100644 results/immich/cadence-rolling-seed/completions/37321739971.json create mode 100644 results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/immich/cadence-rolling-seed/report.json create mode 100644 results/immich/cadence-rolling-seed/report.md create mode 100644 results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json create mode 100644 results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/immich/cadence-rolling-seed/series.json create mode 100644 results/mastodon/cadence-rolling-advance-server/completions/37323126507.json create mode 100644 results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/mastodon/cadence-rolling-advance-server/report.json create mode 100644 results/mastodon/cadence-rolling-advance-server/report.md create mode 100644 results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json create mode 100644 results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json create mode 100644 results/mastodon/cadence-rolling-advance-server/series.json create mode 100644 results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json create mode 100644 results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/mastodon/cadence-rolling-advance-streaming/report.json create mode 100644 results/mastodon/cadence-rolling-advance-streaming/report.md create mode 100644 results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json create mode 100644 results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json create mode 100644 results/mastodon/cadence-rolling-advance-streaming/series.json create mode 100644 results/mastodon/cadence-rolling-seed-server/completions/37321744434.json create mode 100644 results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/mastodon/cadence-rolling-seed-server/report.json create mode 100644 results/mastodon/cadence-rolling-seed-server/report.md create mode 100644 results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json create mode 100644 results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json create mode 100644 results/mastodon/cadence-rolling-seed-server/series.json create mode 100644 results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json create mode 100644 results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/mastodon/cadence-rolling-seed-streaming/report.json create mode 100644 results/mastodon/cadence-rolling-seed-streaming/report.md create mode 100644 results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json create mode 100644 results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json create mode 100644 results/mastodon/cadence-rolling-seed-streaming/series.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/report.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/report.md create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance-02/series.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance/report.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance/report.md create mode 100644 results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-advance/series.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/report.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/report.md create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed-02/series.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed/report.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed/report.md create mode 100644 results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/msgpack-sbt/cadence-rolling-seed/series.json create mode 100644 results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json create mode 100644 results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-advance-distroless/report.json create mode 100644 results/n8n/cadence-rolling-advance-distroless/report.md create mode 100644 results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-distroless/series.json create mode 100644 results/n8n/cadence-rolling-advance-docker/completions/37323145174.json create mode 100644 results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-advance-docker/report.json create mode 100644 results/n8n/cadence-rolling-advance-docker/report.md create mode 100644 results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-docker/series.json create mode 100644 results/n8n/cadence-rolling-advance-runners/completions/37323150431.json create mode 100644 results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-advance-runners/report.json create mode 100644 results/n8n/cadence-rolling-advance-runners/report.md create mode 100644 results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-runners/series.json create mode 100644 results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json create mode 100644 results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-advance-turbo/report.json create mode 100644 results/n8n/cadence-rolling-advance-turbo/report.md create mode 100644 results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-advance-turbo/series.json create mode 100644 results/n8n/cadence-rolling-next-turbo/completions/37325183094.json create mode 100644 results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-next-turbo/report.json create mode 100644 results/n8n/cadence-rolling-next-turbo/report.md create mode 100644 results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-next-turbo/series.json create mode 100644 results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json create mode 100644 results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-seed-distroless/report.json create mode 100644 results/n8n/cadence-rolling-seed-distroless/report.md create mode 100644 results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-distroless/series.json create mode 100644 results/n8n/cadence-rolling-seed-docker/completions/37321758035.json create mode 100644 results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-seed-docker/report.json create mode 100644 results/n8n/cadence-rolling-seed-docker/report.md create mode 100644 results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-docker/series.json create mode 100644 results/n8n/cadence-rolling-seed-runners/completions/37321762921.json create mode 100644 results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-seed-runners/report.json create mode 100644 results/n8n/cadence-rolling-seed-runners/report.md create mode 100644 results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-runners/series.json create mode 100644 results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json create mode 100644 results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/n8n/cadence-rolling-seed-turbo/report.json create mode 100644 results/n8n/cadence-rolling-seed-turbo/report.md create mode 100644 results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json create mode 100644 results/n8n/cadence-rolling-seed-turbo/series.json create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/report.json create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/report.md create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json create mode 100644 results/obs-studio/cadence-rolling-advance-ccache-02/series.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/report.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/report.md create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode-02/series.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode/report.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode/report.md create mode 100644 results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-advance-xcode/series.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/report.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/report.md create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache-02/series.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache/report.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache/report.md create mode 100644 results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-seed-ccache/series.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/report.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/report.md create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode-02/series.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode/report.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode/report.md create mode 100644 results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json create mode 100644 results/obs-studio/cadence-rolling-seed-xcode/series.json create mode 100644 results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json create mode 100644 results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/opencut-moon/cadence-rolling-advance-02/report.json create mode 100644 results/opencut-moon/cadence-rolling-advance-02/report.md create mode 100644 results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json create mode 100644 results/opencut-moon/cadence-rolling-advance-02/series.json create mode 100644 results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json create mode 100644 results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/opencut-moon/cadence-rolling-advance-03/report.json create mode 100644 results/opencut-moon/cadence-rolling-advance-03/report.md create mode 100644 results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json create mode 100644 results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json create mode 100644 results/opencut-moon/cadence-rolling-advance-03/series.json create mode 100644 results/opencut-moon/cadence-rolling-advance/completions/37323159608.json create mode 100644 results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/opencut-moon/cadence-rolling-advance/report.json create mode 100644 results/opencut-moon/cadence-rolling-advance/report.md create mode 100644 results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/opencut-moon/cadence-rolling-advance/series.json create mode 100644 results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json create mode 100644 results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/opencut-moon/cadence-rolling-seed-02/report.json create mode 100644 results/opencut-moon/cadence-rolling-seed-02/report.md create mode 100644 results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json create mode 100644 results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json create mode 100644 results/opencut-moon/cadence-rolling-seed-02/series.json create mode 100644 results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json create mode 100644 results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/opencut-moon/cadence-rolling-seed-03/report.json create mode 100644 results/opencut-moon/cadence-rolling-seed-03/report.md create mode 100644 results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json create mode 100644 results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json create mode 100644 results/opencut-moon/cadence-rolling-seed-03/series.json create mode 100644 results/opencut-moon/cadence-rolling-seed/completions/37322625973.json create mode 100644 results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/opencut-moon/cadence-rolling-seed/report.json create mode 100644 results/opencut-moon/cadence-rolling-seed/report.md create mode 100644 results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json create mode 100644 results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/opencut-moon/cadence-rolling-seed/series.json create mode 100644 results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/posthog/cadence-rolling-advance-layers/series.json create mode 100644 results/posthog/cadence-rolling-seed-layers/completions/37321772493.json create mode 100644 results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json create mode 100644 results/posthog/cadence-rolling-seed-layers/report.json create mode 100644 results/posthog/cadence-rolling-seed-layers/report.md create mode 100644 results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json create mode 100644 results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json create mode 100644 results/posthog/cadence-rolling-seed-layers/series.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/report.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/report.md create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance-02/series.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance/report.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance/report.md create mode 100644 results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-advance/series.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/report.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/report.md create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed-02/series.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed/report.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed/report.md create mode 100644 results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/stackstorm-pants/cadence-rolling-seed/series.json create mode 100644 results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json create mode 100644 results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json create mode 100644 results/zed-nix/cadence-rolling-advance/series.json create mode 100644 results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json create mode 100644 results/zed-nix/cadence-rolling-seed/series.json diff --git a/docs/cadence-qualification-2026-10-05.md b/docs/cadence-qualification-2026-10-05.md new file mode 100644 index 000000000..0a24e9bb4 --- /dev/null +++ b/docs/cadence-qualification-2026-10-05.md @@ -0,0 +1,61 @@ +# Cadence qualification — October 5, 2026 + +Fifteen of seventeen rolling targets have passed a seed run and an actual source +advancement with both providers. PostHog and Zed Nix are still running. This is +an operational correctness review; these single-sample pairs do not establish +comparative performance. All runs use `vcli-canary-7a5b27146ebe`. + +The [machine-readable inventory](../migration/cadence-qualification-2026-10-05.json) +records frozen plans, exact sources, cache scopes and selected run IDs. Earlier +failed and cancelled cohorts remain in `results/`; corrected cohorts have new +plans. The [evidence inventory](../migration/evidence.json) links archives that +were downloaded again and verified after publication. + +| Case / variant | Seed | Changed source | Result | +| --- | --- | --- | --- | +| immich | [37321739971](https://github.com/boringcache/benchmarks/actions/runs/37321739971) | [37323121773](https://github.com/boringcache/benchmarks/actions/runs/37323121773) | Passed | +| mastodon / server | [37321744434](https://github.com/boringcache/benchmarks/actions/runs/37321744434) | [37323126507](https://github.com/boringcache/benchmarks/actions/runs/37323126507) | Passed | +| mastodon / streaming | [37321748690](https://github.com/boringcache/benchmarks/actions/runs/37321748690) | [37323131290](https://github.com/boringcache/benchmarks/actions/runs/37323131290) | Passed | +| n8n / distroless | [37321753318](https://github.com/boringcache/benchmarks/actions/runs/37321753318) | [37323140444](https://github.com/boringcache/benchmarks/actions/runs/37323140444) | Passed | +| n8n / docker | [37321758035](https://github.com/boringcache/benchmarks/actions/runs/37321758035) | [37323145174](https://github.com/boringcache/benchmarks/actions/runs/37323145174) | Passed | +| n8n / runners | [37321762921](https://github.com/boringcache/benchmarks/actions/runs/37321762921) | [37323150431](https://github.com/boringcache/benchmarks/actions/runs/37323150431) | Passed | +| n8n / turbo | [37321767409](https://github.com/boringcache/benchmarks/actions/runs/37321767409) | [37323155106](https://github.com/boringcache/benchmarks/actions/runs/37323155106) | Passed | +| posthog / layers | [37321772493](https://github.com/boringcache/benchmarks/actions/runs/37321772493) | [37323164071](https://github.com/boringcache/benchmarks/actions/runs/37323164071) | Pending | +| executorch-buck2 | [37324032100](https://github.com/boringcache/benchmarks/actions/runs/37324032100) | [37324204556](https://github.com/boringcache/benchmarks/actions/runs/37324204556) | Passed | +| gogs-moon | [37325135541](https://github.com/boringcache/benchmarks/actions/runs/37325135541) | [37325282767](https://github.com/boringcache/benchmarks/actions/runs/37325282767) | Passed | +| opencut-moon | [37325140807](https://github.com/boringcache/benchmarks/actions/runs/37325140807) | [37325287254](https://github.com/boringcache/benchmarks/actions/runs/37325287254) | Passed | +| stackstorm-pants | [37324025714](https://github.com/boringcache/benchmarks/actions/runs/37324025714) | [37324228460](https://github.com/boringcache/benchmarks/actions/runs/37324228460) | Passed | +| msgpack-sbt | [37324036688](https://github.com/boringcache/benchmarks/actions/runs/37324036688) | [37324214890](https://github.com/boringcache/benchmarks/actions/runs/37324214890) | Passed | +| obs-studio / ccache | [37323465858](https://github.com/boringcache/benchmarks/actions/runs/37323465858) | [37324219008](https://github.com/boringcache/benchmarks/actions/runs/37324219008) | Passed | +| obs-studio / xcode | [37326133177](https://github.com/boringcache/benchmarks/actions/runs/37326133177) | [37326340221](https://github.com/boringcache/benchmarks/actions/runs/37326340221) | Passed | +| helix-nix | [37322604229](https://github.com/boringcache/benchmarks/actions/runs/37322604229) | [37323117664](https://github.com/boringcache/benchmarks/actions/runs/37323117664) | Passed | +| zed-nix | [37322636399](https://github.com/boringcache/benchmarks/actions/runs/37322636399) | [37323174641](https://github.com/boringcache/benchmarks/actions/runs/37323174641) | Pending | + +The five REAPI comparator archives identify the intended previous run and source +in `rolling-seed.json`. Both OBS comparator archives likewise identify their +intended seed in `previous-cache-source.json`. The review checks those identities +against the frozen plans; a cache flag alone is insufficient. + +The first n8n Turbo advancement retained an available BoringCache entry but +reported zero native hits. A further actual upstream revision in +[run 37325183094](https://github.com/boringcache/benchmarks/actions/runs/37325183094) +reported 70 native hits and 2 misses. Both observations remain retained. Changed +source does not guarantee a hit; absence of a hit must not be reported as reuse. + +Helix Nix passed two fresh cold/warm samples and its rolling pair. Zed Nix fresh +and rolling qualification remains pending. Each Nix run imports one checksummed +dependency store into its provider workers and checks the dependency baseline. +The measured package is excluded from that seed. A changed package derivation +may require a build even when the rolling provider cache is available. + +The shared controller passed a 24-case hosted planning rehearsal, and fresh +dispatch passed the complete 31-target dry run. The +[publication rehearsal](../migration/rehearsals/mastodon-publication/review.md) +passed isolated conditional publication, dispatch, reconciliation and a forced +concurrent-write conflict. It used the operator token; the first activated cycle +must verify workflow-token writes on `main`. + +Central activation remains off by request until a compatible stable CLI release. +The [cutover procedure](cadence-cutover.md) preserves manual entrypoints and +drains historical runs before central ownership begins. The prepared historical +schedule patches have not been applied. diff --git a/docs/cadence.md b/docs/cadence.md index 188d94ac4..f4e15d602 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -94,7 +94,9 @@ rolling requests with the canary without publishing or dispatching builds. The earlier source matrix completed the cases independently and identified recipe changes in Immich, Qdrant, msgpack and Zed. Their reviewed changes are recorded in [the recipe review](recipe-reviews-2026-10-05.md). New source pins and -changed recipes still require live qualification. +changed recipes are covered by the +[live qualification review](cadence-qualification-2026-10-05.md), which retains +successful, failed and pending observations separately. The repository's Actions cache limit is 200 GB, with seven-day retention. Capacity alone does not establish that a particular rolling seed was retained @@ -102,9 +104,9 @@ or restored. Remaining activation requirements: -- Pass hosted checks for the complete controller and rolling changes. -- Qualify the new and changed rolling paths with actual source advancement, - retained seeds, verified outputs and canary CLI execution. +- Finish PostHog and Zed Nix rolling qualification and evidence preservation. + The other fifteen rolling targets have passed seed and changed-source runs; + hosted checks and 252 local tests pass. - Pass the corrected Zed Nix cold/warm screen using the common dependency seed. - Verify main-branch workflow-token publication at cutover. The [isolated publication rehearsal](../migration/rehearsals/mastodon-publication/review.md) diff --git a/migration/cadence-qualification-2026-10-05.json b/migration/cadence-qualification-2026-10-05.json new file mode 100644 index 000000000..c9c846381 --- /dev/null +++ b/migration/cadence-qualification-2026-10-05.json @@ -0,0 +1,936 @@ +{ + "schema_version": 1, + "reviewed_on": "2026-10-05", + "cli_version": "vcli-canary-7a5b27146ebe", + "cases": 12, + "targets": 17, + "activation": "deferred until a compatible stable CLI release and schedule cutover", + "observations": "All executed earlier cohorts remain retained, including failed and cancelled runs. Corrected cohorts are separate frozen plans.", + "rolling": [ + { + "case_id": "immich", + "variant": null, + "seed": { + "series": "results/immich/cadence-rolling-seed", + "run_id": 37321739971, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-immich", + "completion": "success" + }, + "advance": { + "series": "results/immich/cadence-rolling-advance", + "run_id": 37323121773, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-immich", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "mastodon", + "variant": "server", + "seed": { + "series": "results/mastodon/cadence-rolling-seed-server", + "run_id": 37321744434, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-server", + "completion": "success" + }, + "advance": { + "series": "results/mastodon/cadence-rolling-advance-server", + "run_id": 37323126507, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-server", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "mastodon", + "variant": "streaming", + "seed": { + "series": "results/mastodon/cadence-rolling-seed-streaming", + "run_id": 37321748690, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-streaming", + "completion": "success" + }, + "advance": { + "series": "results/mastodon/cadence-rolling-advance-streaming", + "run_id": 37323131290, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-mastodon-streaming", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "distroless", + "seed": { + "series": "results/n8n/cadence-rolling-seed-distroless", + "run_id": 37321753318, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-distroless", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-distroless", + "run_id": 37323140444, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-distroless", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "docker", + "seed": { + "series": "results/n8n/cadence-rolling-seed-docker", + "run_id": 37321758035, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-docker", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-docker", + "run_id": 37323145174, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-docker", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "runners", + "seed": { + "series": "results/n8n/cadence-rolling-seed-runners", + "run_id": 37321762921, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-runners", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-runners", + "run_id": 37323150431, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-runners", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "n8n", + "variant": "turbo", + "seed": { + "series": "results/n8n/cadence-rolling-seed-turbo", + "run_id": 37321767409, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-turbo", + "completion": "success" + }, + "advance": { + "series": "results/n8n/cadence-rolling-advance-turbo", + "run_id": 37323155106, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-turbo", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "additional_advancement": { + "series": "results/n8n/cadence-rolling-next-turbo", + "run_id": 37325183094, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-n8n-turbo", + "completion": "success" + }, + "reuse_observation": "The first advancement retained a usable BoringCache entry but reported zero native hits. The next actual upstream revision reported 70 native hits and 2 misses. Both observations are retained." + }, + { + "case_id": "posthog", + "variant": "layers", + "seed": { + "series": "results/posthog/cadence-rolling-seed-layers", + "run_id": 37321772493, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-posthog-layers", + "completion": "success" + }, + "advance": { + "series": "results/posthog/cadence-rolling-advance-layers", + "run_id": 37323164071, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "cache_scope": "cadence-20261005-posthog-layers", + "completion": "awaiting-completion-evidence" + }, + "state": "pending", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "executorch-buck2", + "variant": null, + "seed": { + "series": "results/executorch-buck2/cadence-rolling-seed-02", + "run_id": 37324032100, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "completion": "success" + }, + "advance": { + "series": "results/executorch-buck2/cadence-rolling-advance-02", + "run_id": 37324204556, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-executorch-buck2-cadence-20261005-executorch-buck2-v2-Linux-X64-37324032100-1", + "previous": { + "case_id": "executorch-buck2", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324032100", + "run_attempt": "1" + }, + "current": { + "case_id": "executorch-buck2", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324204556", + "run_attempt": "1" + } + } + }, + { + "case_id": "gogs-moon", + "variant": null, + "seed": { + "series": "results/gogs-moon/cadence-rolling-seed-03", + "run_id": 37325135541, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "cache_scope": "cadence-20261005-gogs-moon-v3", + "completion": "success" + }, + "advance": { + "series": "results/gogs-moon/cadence-rolling-advance-03", + "run_id": 37325282767, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "cache_scope": "cadence-20261005-gogs-moon-v3", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-gogs-moon-cadence-20261005-gogs-moon-v3-Linux-X64-37325135541-1", + "previous": { + "case_id": "gogs-moon", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "run_id": "37325135541", + "run_attempt": "1" + }, + "current": { + "case_id": "gogs-moon", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "run_id": "37325282767", + "run_attempt": "1" + } + } + }, + { + "case_id": "opencut-moon", + "variant": null, + "seed": { + "series": "results/opencut-moon/cadence-rolling-seed-03", + "run_id": 37325140807, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "cache_scope": "cadence-20261005-opencut-moon-v3", + "completion": "success" + }, + "advance": { + "series": "results/opencut-moon/cadence-rolling-advance-03", + "run_id": 37325287254, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "cache_scope": "cadence-20261005-opencut-moon-v3", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-opencut-moon-cadence-20261005-opencut-moon-v3-Linux-X64-37325140807-1", + "previous": { + "case_id": "opencut-moon", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "run_id": "37325140807", + "run_attempt": "1" + }, + "current": { + "case_id": "opencut-moon", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "run_id": "37325287254", + "run_attempt": "1" + } + } + }, + { + "case_id": "stackstorm-pants", + "variant": null, + "seed": { + "series": "results/stackstorm-pants/cadence-rolling-seed-02", + "run_id": 37324025714, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "completion": "success" + }, + "advance": { + "series": "results/stackstorm-pants/cadence-rolling-advance-02", + "run_id": 37324228460, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-stackstorm-pants-cadence-20261005-stackstorm-pants-v2-Linux-X64-37324025714-1", + "previous": { + "case_id": "stackstorm-pants", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "run_id": "37324025714", + "run_attempt": "1" + }, + "current": { + "case_id": "stackstorm-pants", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "run_id": "37324228460", + "run_attempt": "1" + } + } + }, + { + "case_id": "msgpack-sbt", + "variant": null, + "seed": { + "series": "results/msgpack-sbt/cadence-rolling-seed-02", + "run_id": 37324036688, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "completion": "success" + }, + "advance": { + "series": "results/msgpack-sbt/cadence-rolling-advance-02", + "run_id": 37324214890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "cache_key": "reapi-msgpack-sbt-cadence-20261005-msgpack-sbt-v2-Linux-X64-37324036688-1", + "previous": { + "case_id": "msgpack-sbt", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324036688", + "run_attempt": "1" + }, + "current": { + "case_id": "msgpack-sbt", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "run_id": "37324214890", + "run_attempt": "1" + } + } + }, + { + "case_id": "obs-studio", + "variant": "ccache", + "seed": { + "series": "results/obs-studio/cadence-rolling-seed-ccache-02", + "run_id": 37323465858, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "completion": "success" + }, + "advance": { + "series": "results/obs-studio/cadence-rolling-advance-ccache-02", + "run_id": 37324219008, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "source_sha": "ef5814431c6af9061874470201d0b799bc84602b", + "run_id": "37323465858", + "run_attempt": "1" + } + }, + { + "case_id": "obs-studio", + "variant": "xcode", + "seed": { + "series": "results/obs-studio/cadence-rolling-seed-xcode-02", + "run_id": 37326133177, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "completion": "success" + }, + "advance": { + "series": "results/obs-studio/cadence-rolling-advance-xcode-02", + "run_id": 37326340221, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification", + "comparator_lineage": { + "source_sha": "ef5814431c6af9061874470201d0b799bc84602b", + "run_id": "37326133177", + "run_attempt": "1" + } + }, + { + "case_id": "helix-nix", + "variant": null, + "seed": { + "series": "results/helix-nix/cadence-rolling-seed", + "run_id": 37322604229, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "cache_scope": "cadence-20261005-helix-nix", + "completion": "success" + }, + "advance": { + "series": "results/helix-nix/cadence-rolling-advance", + "run_id": 37323117664, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "cache_scope": "cadence-20261005-helix-nix", + "completion": "success" + }, + "state": "executions-passed", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + }, + { + "case_id": "zed-nix", + "variant": null, + "seed": { + "series": "results/zed-nix/cadence-rolling-seed", + "run_id": 37322636399, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "cache_scope": "cadence-20261005-zed-nix", + "completion": "awaiting-completion-evidence" + }, + "advance": { + "series": "results/zed-nix/cadence-rolling-advance", + "run_id": 37323174641, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "cache_scope": "cadence-20261005-zed-nix", + "completion": "awaiting-completion-evidence" + }, + "state": "pending", + "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" + } + ] +} diff --git a/migration/evidence.json b/migration/evidence.json index cf2059a76..ecbf063c9 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -1667,6 +1667,762 @@ "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", "verified_on": "2026-10-05", "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37315116541, + "verified_files": 9, + "gaps": [], + "sha256": "ca9909f44c6315270a806f95c2d2707e64e3add3d79fb6415722097b0b3d0ba5", + "bundle_bytes": 31037, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37315116541.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37320867908, + "verified_files": 22, + "gaps": [], + "sha256": "f7a9664a0b17b2d2d448c99710d15514f1e30545bcfaa43fa9b34956379d2b3f", + "bundle_bytes": 1101686405, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320867908.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321566845, + "verified_files": 48, + "gaps": [], + "sha256": "a692391d0263c7dd9de5f4da74afbc2b7e71f9ad73ee9b0fa3ec1c687800b974", + "bundle_bytes": 1097296, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321566845.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321739971, + "verified_files": 12, + "gaps": [], + "sha256": "8d242f66c0eb47d5a45b0ae82c6a87f072820fb2b21a85053633a0000d805601", + "bundle_bytes": 275449, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321739971.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321744434, + "verified_files": 12, + "gaps": [], + "sha256": "cfad4bc4a1ea4794335742bf5f7b72068df1be53f5829588e0be7e0be5ef703b", + "bundle_bytes": 827657, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321744434.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321748690, + "verified_files": 12, + "gaps": [], + "sha256": "fec6d35de4f43c5107db22a06d8709872f9b0974845b2c61622ba08f4540d9d6", + "bundle_bytes": 156444, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321748690.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321753318, + "verified_files": 12, + "gaps": [], + "sha256": "943f1aed20b84a20ae7bd3e66c867e9aa377770748c091f494f890265b4acf15", + "bundle_bytes": 1539621, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321753318.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321758035, + "verified_files": 12, + "gaps": [], + "sha256": "604aca8af1d1211ccdf93bb62aa955eca12b949000b647309e76f75889890b73", + "bundle_bytes": 1595003, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321758035.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321762921, + "verified_files": 12, + "gaps": [], + "sha256": "099866cf3894bf3f24d99237c56d567e9a891d69795a3618b6db55f81fe248e0", + "bundle_bytes": 1469723, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321762921.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321767409, + "verified_files": 12, + "gaps": [], + "sha256": "333d355c184ced314904c188faacd8c8f128ca3009af4b07eedfdbfbdf5e3f7c", + "bundle_bytes": 329937, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321767409.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37321772493, + "verified_files": 12, + "gaps": [], + "sha256": "8ea634086f30e9d31421ee5001759f28d6a5abb1835a8663c786ccc2bf420cb0", + "bundle_bytes": 347721, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37321772493.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322592957, + "verified_files": 14, + "gaps": [], + "sha256": "7f67f53fa6cbfdf9e84fb54b78bdf084f38d79294e2279f671cba2df13e6053c", + "bundle_bytes": 191113, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322592957.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322598011, + "verified_files": 14, + "gaps": [], + "sha256": "d2bb19c69a726dafafb9cc998a8c54c0a442e6c2225bffc6d5d95eb0d4883acf", + "bundle_bytes": 220421, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322598011.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322604229, + "verified_files": 17, + "gaps": [], + "sha256": "592819345698de92cb117fdfa12e5e503830e3d477686667057c535450b9bf9f", + "bundle_bytes": 1100939913, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322604229.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322609279, + "verified_files": 14, + "gaps": [], + "sha256": "b150fa6c1b18923d658c39b2682f9bec768c99c58b3e47c3436e7267e4924507", + "bundle_bytes": 326834, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322609279.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322616359, + "verified_files": 11, + "gaps": [], + "sha256": "763f7a67eeb11a7709d74e2428ff656a0ede56664a81e7088dbe61cf6a42a798", + "bundle_bytes": 338991, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322616359.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322621030, + "verified_files": 12, + "gaps": [], + "sha256": "bffc77b9bc836074a77383fe5ef6ff5974def9d16ce6600c6f3a17c3041a1572", + "bundle_bytes": 366811, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322621030.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322625973, + "verified_files": 14, + "gaps": [], + "sha256": "82ec59c02b6341663b1dace65e372ff15dea6c9bb727b179402a2e4e643eb7ec", + "bundle_bytes": 115482, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322625973.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322630658, + "verified_files": 14, + "gaps": [], + "sha256": "bd5238de08f55c051c10af54129672dd68a321eb5767b76bf378a7c9c35076d5", + "bundle_bytes": 113197, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322630658.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322979117, + "verified_files": 12, + "gaps": [], + "sha256": "67c1ce2b89ccb76d94bb7e0b3bce59bde293ca308c17cc33b991f75f6a7eb933", + "bundle_bytes": 826739, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322979117.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322983836, + "verified_files": 12, + "gaps": [], + "sha256": "cb6fe435b0385ecbd9bf908a995caca4057219763a1a40cf32a7235aa731911d", + "bundle_bytes": 154007, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322983836.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323107836, + "verified_files": 9, + "gaps": [], + "sha256": "c03c24b601c1a8e5bb5665974244c8a1909a8c4baa39fef7af98e24135fbf668", + "bundle_bytes": 84529, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323107836.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323112601, + "verified_files": 11, + "gaps": [], + "sha256": "cd0ec90eec324ba58a7d0057e07f88479d748b06f2e2767e95a24c7e84e36a9c", + "bundle_bytes": 145242, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323112601.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323117664, + "verified_files": 17, + "gaps": [], + "sha256": "a6c81e98459285213015dcab97e58811e265b2ea14b05a14033bed8096822780", + "bundle_bytes": 1101082613, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323117664.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323121773, + "verified_files": 12, + "gaps": [], + "sha256": "5801a2d2a574d29f3b9bf57a498b8affc4149fa9e13b799cd654c4da24cde71c", + "bundle_bytes": 282104, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323121773.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323126507, + "verified_files": 12, + "gaps": [], + "sha256": "1ad5f7e4ca2684465c84f7dd8718aae2278ff16fbdcefd190771cb6eb000dbe9", + "bundle_bytes": 435959, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323126507.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323131290, + "verified_files": 12, + "gaps": [], + "sha256": "61e572577342afc628eeefd6991964b60d30f6e309365f8567447366ff6e6bea", + "bundle_bytes": 146637, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323131290.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323136088, + "verified_files": 11, + "gaps": [], + "sha256": "72b85d786bdf715a8a1e2c9ca796a8100a1115cdc6d33ad50c5172a7a5ed429c", + "bundle_bytes": 173633, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323136088.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323140444, + "verified_files": 12, + "gaps": [], + "sha256": "df570b5c90c61cc8361a3417b07993243388a5b14027680c0a95b842488293ee", + "bundle_bytes": 1521595, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323140444.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323145174, + "verified_files": 12, + "gaps": [], + "sha256": "a9eb1b54d32882655857258e64fd3502748dba73d6dbc965d4e002db0fc9320f", + "bundle_bytes": 1492015, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323145174.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323150431, + "verified_files": 12, + "gaps": [], + "sha256": "7b9e55505c6e141641973baabe03b5d10601bcd96902f4a198492212ef8cd750", + "bundle_bytes": 1494733, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323150431.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323155106, + "verified_files": 12, + "gaps": [], + "sha256": "92d86e6093a84762c6f520e492db177f37e367f8707a284d528a211791c2efc6", + "bundle_bytes": 342132, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323155106.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323159608, + "verified_files": 11, + "gaps": [], + "sha256": "529277911178291c805dcd36b9e2b05c5957f94125071f76db212acd73dfd136", + "bundle_bytes": 100977, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323159608.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323169828, + "verified_files": 11, + "gaps": [], + "sha256": "c401fac1f49c571d64e22854bebbcbbacd68f625c7b6054994da260487c22461", + "bundle_bytes": 97016, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323169828.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323234591, + "verified_files": 12, + "gaps": [], + "sha256": "004555369daa75edae6aa4f987b1a05f54145d02e2e6bb8acd0604aca64643d2", + "bundle_bytes": 359874, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323234591.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323465858, + "verified_files": 14, + "gaps": [], + "sha256": "53ff9d72bdff5910a049abd83288b595b4776db3296f0913f432fb46a6240b6b", + "bundle_bytes": 556245, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323465858.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324016178, + "verified_files": 14, + "gaps": [], + "sha256": "7c2234a43786944d100dd1a4af90e8394e94484ecd6936727223fc04f070402e", + "bundle_bytes": 212147, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324016178.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324020999, + "verified_files": 14, + "gaps": [], + "sha256": "6eb956103b370f83cb0e254de9536acb03938442ff1b62496e1e96ed37ef2676", + "bundle_bytes": 107468, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324020999.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324025714, + "verified_files": 14, + "gaps": [], + "sha256": "c0d5b3968fd3baef55c41533cc727b127ff5889094cd510e50f8db4f6f43ae18", + "bundle_bytes": 104968, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324025714.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324032100, + "verified_files": 14, + "gaps": [], + "sha256": "cab8cfbc72be15f995b06274a9b01d24f0fb219773c288700c10b7f266cb387a", + "bundle_bytes": 183124, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324032100.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324036688, + "verified_files": 14, + "gaps": [], + "sha256": "1f47fb24f0bebbb20f5c74433ba379aa9908839ee9e2af384404ca1349d5e8fc", + "bundle_bytes": 318399, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324036688.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324204556, + "verified_files": 14, + "gaps": [], + "sha256": "4fc9961f582cef0bce623ead2d3931b4bdc98e5169d35dff8bbae939285f84b8", + "bundle_bytes": 161465, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324204556.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324209639, + "verified_files": 10, + "gaps": [], + "sha256": "9221d4046fb51bf1ebf63b18851ff4ea7e51cb968ea41cedb8dfea7b8ee38158", + "bundle_bytes": 153230, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324209639.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324214890, + "verified_files": 14, + "gaps": [], + "sha256": "ebeda092c2c2ff2c299f2ea0e4aabb119796f5f3e718bf40c6bc1fe3291e496f", + "bundle_bytes": 258679, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324214890.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324219008, + "verified_files": 14, + "gaps": [], + "sha256": "d4fed72d12b3a4192f88aa482ae3704467909296d734d4ebed6dc898112e4d89", + "bundle_bytes": 550952, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324219008.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324223748, + "verified_files": 10, + "gaps": [], + "sha256": "cad3872631c0ee88d449536487e595e30742ee6d5d0a07b3fca8a885ea58bca3", + "bundle_bytes": 95227, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324223748.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37324228460, + "verified_files": 14, + "gaps": [], + "sha256": "8c51c61558e0ff24b43c70ee64991636577e90408ce3db6e7472f91d603b68e6", + "bundle_bytes": 106198, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37324228460.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325135541, + "verified_files": 14, + "gaps": [], + "sha256": "62fb0f146fa5feb568034f503c814b5aba1016805872c1a16500a668be352c94", + "bundle_bytes": 209949, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325135541.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325140807, + "verified_files": 14, + "gaps": [], + "sha256": "cc1af7a88f60ba75173a25bb0ffad5e6081816b66234b3529c589f2acff7e0be", + "bundle_bytes": 103962, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325140807.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325183094, + "verified_files": 12, + "gaps": [], + "sha256": "c38518b43c2552fca4c2300c251f48f84da443d1fd216c24dfdf012f65a9123a", + "bundle_bytes": 327800, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325183094.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325282767, + "verified_files": 14, + "gaps": [], + "sha256": "35f5c6ae08f20dddebe4d2520b0341f30f1f79106de6e8fb520eb1244c7862bf", + "bundle_bytes": 211243, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325282767.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325287254, + "verified_files": 14, + "gaps": [], + "sha256": "2223af0b476341716133ce0221e569b55c073ead9304f21ea5eda60c70d93b33", + "bundle_bytes": 105379, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325287254.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37326133177, + "verified_files": 14, + "gaps": [], + "sha256": "11526bdc0cf3d01e989928b3ee6aef7d7636bd2ad495ff1b1f769abde866019d", + "bundle_bytes": 363454, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37326133177.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37326340221, + "verified_files": 14, + "gaps": [], + "sha256": "0eb74a37aaa8137d5592c88eadd00cfc406afb40a7cf6128991dbe15a0115c20", + "bundle_bytes": 361816, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37326340221.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" } ] } diff --git a/results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json b/results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json new file mode 100644 index 000000000..43c3d5027 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/completions/37324204556.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324204556", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "cabad803e46d684f98d37e05d6725817a86d6d9e279f8b49a7185bb3e99312d6" +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..d855b69ef --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324204556, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556" +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/report.json b/results/executorch-buck2/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..d3c3b8096 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "f73f472febc211a9b27ff0d819c5a6a25c6db9a28aa614f194ef7c1e5f043650", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324204556", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324204556", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "cabad803e46d684f98d37e05d6725817a86d6d9e279f8b49a7185bb3e99312d6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10076324300001716, + "build_seconds": 27.035207954000043, + "workflow_seconds": null, + "total_seconds": 27.13597119700006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003453705999959311, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.13597119700006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174578", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] + }, + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20143210499998077, + "build_seconds": 21.858722754999974, + "workflow_seconds": null, + "total_seconds": 22.060154859999955, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9866127600000141, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 22.060154859999955 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174579", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/report.md b/results/executorch-buck2/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..762d40cae --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/report.md @@ -0,0 +1,30 @@ +# executorch-buck2: cadence-rolling-advance-02 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10076324300001716 | 27.035207954000043 | 27.13597119700006 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20143210499998077 | 21.858722754999974 | 22.060154859999955 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..50f7f78f3 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10076324300001716, + "build_seconds": 27.035207954000043, + "workflow_seconds": null, + "total_seconds": 27.13597119700006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003453705999959311, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.13597119700006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174578", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f97fcd1d4 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20143210499998077, + "build_seconds": 21.858722754999974, + "workflow_seconds": null, + "total_seconds": 22.060154859999955, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9866127600000141, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 22.060154859999955 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324204556", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174579", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324204556-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324204556" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-advance-02/series.json b/results/executorch-buck2/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..27531ffea --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "1d0153109c29c012c85aa18d5f2fc12872083df4650827a5ef18f390888af24d", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "f73f472febc211a9b27ff0d819c5a6a25c6db9a28aa614f194ef7c1e5f043650" +} diff --git a/results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json b/results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json new file mode 100644 index 000000000..c4d5e8438 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/completions/37323107836.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323107836", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled", + "bazel-remote executorch-buck2 commit: cancelled", + "BoringCache executorch-buck2 commit: cancelled", + "${{ inputs.case_id }} rolling report: cancelled" + ], + "evidence_manifest_sha256": "8560277d3446809f06cdfa9491df840954477e4305218e6acee783a31e19d1ea" +} diff --git a/results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..2564b5b40 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:28Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323107836, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836" +} diff --git a/results/executorch-buck2/cadence-rolling-advance/report.json b/results/executorch-buck2/cadence-rolling-advance/report.json new file mode 100644 index 000000000..7f6149c0b --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/report.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "plan_sha256": "b5a60430a6aea0bbc4b78b616f032125887f7f10628313235f9d316c4919900f", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323107836", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled", + "bazel-remote executorch-buck2 commit: cancelled", + "BoringCache executorch-buck2 commit: cancelled", + "${{ inputs.case_id }} rolling report: cancelled" + ], + "evidence_manifest_sha256": "8560277d3446809f06cdfa9491df840954477e4305218e6acee783a31e19d1ea" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323107836", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323107836", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled", + "bazel-remote executorch-buck2 commit: cancelled", + "BoringCache executorch-buck2 commit: cancelled", + "${{ inputs.case_id }} rolling report: cancelled" + ], + "evidence_manifest_sha256": "8560277d3446809f06cdfa9491df840954477e4305218e6acee783a31e19d1ea" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "boringcache" + ], + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-advance/report.md b/results/executorch-buck2/cadence-rolling-advance/report.md new file mode 100644 index 000000000..62dd7a5e2 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# executorch-buck2: cadence-rolling-advance + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 0/2 recorded; 0 failed; 2 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323107836 failed completion checks: + +- Workflow concluded cancelled +- bazel-remote executorch-buck2 commit: cancelled +- BoringCache executorch-buck2 commit: cancelled +- ${{ inputs.case_id }} rolling report: cancelled + +Missing observations: + +- Sample 1, BoringCache, Changed-source build +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-advance/series.json b/results/executorch-buck2/cadence-rolling-advance/series.json new file mode 100644 index 000000000..6f2ebf22e --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-advance/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e022775af4acc3da2ff2916716d1ec5f3f7d6021bc07325f91e703b93af37647", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b5a60430a6aea0bbc4b78b616f032125887f7f10628313235f9d316c4919900f" +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json b/results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json new file mode 100644 index 000000000..1662ae814 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/completions/37324032100.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324032100", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3e4e6174c3c5bff4a09c840f4d8d17aa6dfc60f48a6100b307938ea86811f13c" +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..5eed2dfe7 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:22Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324032100, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100" +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/report.json b/results/executorch-buck2/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..77610a6c9 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "faeaeb496d8118d98a9bad0fe2fb23a693f95f964e36df35d46b3e0175e0a981", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324032100", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324032100", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3e4e6174c3c5bff4a09c840f4d8d17aa6dfc60f48a6100b307938ea86811f13c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010354490000509, + "build_seconds": 158.649545483, + "workflow_seconds": null, + "total_seconds": 158.75058093200005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004191613999978472, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 158.75058093200005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174533", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] + }, + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20147264299998824, + "build_seconds": 161.122134534, + "workflow_seconds": null, + "total_seconds": 161.32360717699999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 2.3798232430000326, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 161.32360717699999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174532", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/report.md b/results/executorch-buck2/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..5375f7d9f --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# executorch-buck2: cadence-rolling-seed-02 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1010354490000509 | 158.649545483 | 158.75058093200005 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20147264299998824 | 161.122134534 | 161.32360717699999 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..a2ff00677 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010354490000509, + "build_seconds": 158.649545483, + "workflow_seconds": null, + "total_seconds": 158.75058093200005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004191613999978472, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 158.75058093200005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174533", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..8ba8054a1 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20147264299998824, + "build_seconds": 161.122134534, + "workflow_seconds": null, + "total_seconds": 161.32360717699999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 2.3798232430000326, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 161.32360717699999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324032100", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174532", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324032100-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324032100" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed-02/series.json b/results/executorch-buck2/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..eaa6d8733 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "5a55af52841ad83c894439ac4c45b088d5355db883afc07d6a0db5b7a62957c8", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2-v2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "faeaeb496d8118d98a9bad0fe2fb23a693f95f964e36df35d46b3e0175e0a981" +} diff --git a/results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json b/results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json new file mode 100644 index 000000000..cd1f43ade --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/completions/37322592957.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322592957", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322592957", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ef10d1633ac3cbe6e9b49652f2d4e421147ab1558ec026da88982303d2dc9521" +} diff --git a/results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..269294476 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:36Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322592957, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322592957" +} diff --git a/results/executorch-buck2/cadence-rolling-seed/report.json b/results/executorch-buck2/cadence-rolling-seed/report.json new file mode 100644 index 000000000..bc0d07159 --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "35d1e5b1e7d52193a77c09c26dfc6b92ffdd4d61738a3c856ff4c5245ee3792e", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322592957", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322592957", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ef10d1633ac3cbe6e9b49652f2d4e421147ab1558ec026da88982303d2dc9521" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102946500001053, + "build_seconds": 163.48988875000003, + "workflow_seconds": null, + "total_seconds": 163.59091821500004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.005228078999948593, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 163.59091821500004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174431", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] + }, + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20142720100000133, + "build_seconds": 127.76391611900004, + "workflow_seconds": null, + "total_seconds": 127.96534332000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 3.117719557999976, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 127.96534332000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174430", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/cadence-rolling-seed/report.md b/results/executorch-buck2/cadence-rolling-seed/report.md new file mode 100644 index 000000000..0a27801da --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# executorch-buck2: cadence-rolling-seed + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10102946500001053 | 163.48988875000003 | 163.59091821500004 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20142720100000133 | 127.76391611900004 | 127.96534332000004 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6f088288f --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102946500001053, + "build_seconds": 163.48988875000003, + "workflow_seconds": null, + "total_seconds": 163.59091821500004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.005228078999948593, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 163.59091821500004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174431", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..16901156b --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20142720100000133, + "build_seconds": 127.76391611900004, + "workflow_seconds": null, + "total_seconds": 127.96534332000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 3.117719557999976, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 127.96534332000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-rolling-cadence-20261005-executorch-buck2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "91d26b3140533172f2557eabb1476e22f0b98bc3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322592957", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174430", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322592957-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322592957" + ] +} diff --git a/results/executorch-buck2/cadence-rolling-seed/series.json b/results/executorch-buck2/cadence-rolling-seed/series.json new file mode 100644 index 000000000..3989fe1dd --- /dev/null +++ b/results/executorch-buck2/cadence-rolling-seed/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "753ba0c6b6c4e806bb24b21efb5f1c098da6c931155b4b45f84bb392ba43a8c3", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-executorch-buck2" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "91d26b3140533172f2557eabb1476e22f0b98bc3" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "35d1e5b1e7d52193a77c09c26dfc6b92ffdd4d61738a3c856ff4c5245ee3792e" +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json b/results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json new file mode 100644 index 000000000..1379d1357 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/completions/37324209639.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324209639", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "c6f067228af8803ce9918774f33c07012c144d6c60330777002c855ff6af8117" +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..1856eb5e5 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324209639, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639" +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/report.json b/results/gogs-moon/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..696acf629 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/report.json @@ -0,0 +1,220 @@ +{ + "schema_version": 1, + "plan_sha256": "5660c43a1cd7acddbb845be324a33442a7786c3299af0b1d2ac0fc78a7ddec6b", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324209639", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "c6f067228af8803ce9918774f33c07012c144d6c60330777002c855ff6af8117" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324209639", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324209639", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "c6f067228af8803ce9918774f33c07012c144d6c60330777002c855ff6af8117" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011285519999916, + "build_seconds": 8.481873947000011, + "workflow_seconds": null, + "total_seconds": 8.583002499000003, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001908086000000253, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.583002499000003 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324209639", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174556", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324209639-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324209639" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/report.md b/results/gogs-moon/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..3094b44ed --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/report.md @@ -0,0 +1,38 @@ +# gogs-moon: cadence-rolling-advance-02 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37324209639 failed completion checks: + +- Workflow concluded failure +- BoringCache gogs-moon commit: failure +- gogs-moon rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1011285519999916 | 8.481873947000011 | 8.583002499000003 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6564d993f --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011285519999916, + "build_seconds": 8.481873947000011, + "workflow_seconds": null, + "total_seconds": 8.583002499000003, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001908086000000253, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.583002499000003 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324209639", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174556", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324209639-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324209639" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance-02/series.json b/results/gogs-moon/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..51cd9834d --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "5660c43a1cd7acddbb845be324a33442a7786c3299af0b1d2ac0fc78a7ddec6b" +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json b/results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json new file mode 100644 index 000000000..1c657b93b --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/completions/37325282767.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325282767", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2f19fc35799b4b7972eae3a033bd77e7f64127c99f6bcb801e171cfe6e8cce18" +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..ed00d000b --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3", + "series_id": "cadence-rolling-advance-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:30:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325282767, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767" +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/report.json b/results/gogs-moon/cadence-rolling-advance-03/report.json new file mode 100644 index 000000000..2fe3586f2 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "7d4cffbb92d6bdc71d107a9bd62991f93d0ee589cc6fa0914905f21e7aeb1517", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325282767", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325282767", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2f19fc35799b4b7972eae3a033bd77e7f64127c99f6bcb801e171cfe6e8cce18" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10085198400000195, + "build_seconds": 7.968215224000005, + "workflow_seconds": null, + "total_seconds": 8.069067208000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001377290000007747, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.069067208000007 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174605", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20138485300000752, + "build_seconds": 8.271418910000008, + "workflow_seconds": null, + "total_seconds": 8.472803763000016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9762034230000012, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.472803763000016 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174606", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/report.md b/results/gogs-moon/cadence-rolling-advance-03/report.md new file mode 100644 index 000000000..9da06fc04 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-advance-03 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10085198400000195 | 7.968215224000005 | 8.069067208000007 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20138485300000752 | 8.271418910000008 | 8.472803763000016 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..0ef168660 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10085198400000195, + "build_seconds": 7.968215224000005, + "workflow_seconds": null, + "total_seconds": 8.069067208000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001377290000007747, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.069067208000007 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174605", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f771aa06c --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20138485300000752, + "build_seconds": 8.271418910000008, + "workflow_seconds": null, + "total_seconds": 8.472803763000016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9762034230000012, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.472803763000016 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325282767", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174606", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325282767-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325282767" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance-03/series.json b/results/gogs-moon/cadence-rolling-advance-03/series.json new file mode 100644 index 000000000..6d35b8e64 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance-03", + "created_at": "2026-10-05T14:30:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "7913463422e6c7956ff1909d3a1fbc42817922b569f34b1bc35bd33d2ece44a9", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "7d4cffbb92d6bdc71d107a9bd62991f93d0ee589cc6fa0914905f21e7aeb1517" +} diff --git a/results/gogs-moon/cadence-rolling-advance/completions/37323112601.json b/results/gogs-moon/cadence-rolling-advance/completions/37323112601.json new file mode 100644 index 000000000..83dc4a000 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/completions/37323112601.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323112601", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "2a499070ff217a6ad4e09274fac835824cb2c065a27d45aaf1efc4fc8c16483a" +} diff --git a/results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..6108a9bad --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:30Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323112601, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601" +} diff --git a/results/gogs-moon/cadence-rolling-advance/report.json b/results/gogs-moon/cadence-rolling-advance/report.json new file mode 100644 index 000000000..9578ef243 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "c7a436522dc1526a276da2991f093d32c4bf1440c6ad9110ad08ca2f7593e412", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323112601", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "2a499070ff217a6ad4e09274fac835824cb2c065a27d45aaf1efc4fc8c16483a" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323112601", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323112601", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote gogs-moon commit: failure", + "gogs-moon rolling report: failure" + ], + "evidence_manifest_sha256": "2a499070ff217a6ad4e09274fac835824cb2c065a27d45aaf1efc4fc8c16483a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "abb49ac3a869811c5c19b69223e824cdc75f2921c309473b6ec924dda803bf89", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10094187800001464, + "build_seconds": 8.97829006400002, + "workflow_seconds": null, + "total_seconds": 9.079231942000035, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.5973653270000341, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.079231942000035 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323112601", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174469", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323112601-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323112601" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-advance/report.md b/results/gogs-moon/cadence-rolling-advance/report.md new file mode 100644 index 000000000..b4f0f3b10 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# gogs-moon: cadence-rolling-advance + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323112601 failed completion checks: + +- Workflow concluded failure +- bazel-remote gogs-moon commit: failure +- gogs-moon rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.10094187800001464 | 8.97829006400002 | 9.079231942000035 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..839227812 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "abb49ac3a869811c5c19b69223e824cdc75f2921c309473b6ec924dda803bf89", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10094187800001464, + "build_seconds": 8.97829006400002, + "workflow_seconds": null, + "total_seconds": 9.079231942000035, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.5973653270000341, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.079231942000035 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323112601", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174469", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323112601-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323112601" + ] +} diff --git a/results/gogs-moon/cadence-rolling-advance/series.json b/results/gogs-moon/cadence-rolling-advance/series.json new file mode 100644 index 000000000..adba0dc27 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-advance/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "abb49ac3a869811c5c19b69223e824cdc75f2921c309473b6ec924dda803bf89", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "c7a436522dc1526a276da2991f093d32c4bf1440c6ad9110ad08ca2f7593e412" +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json b/results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json new file mode 100644 index 000000000..8fc6e7d05 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/completions/37324016178.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324016178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324016178", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "564fa89fa147d542879d4ad7d8b9564103bc572af9da1acd609a18d2804ccf21" +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..ac1b7e9ca --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:15Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324016178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324016178" +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/report.json b/results/gogs-moon/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..d336db5b3 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "23b7faf2d7426940a8b1dc9065cf32fc3bc7adac6f2ce1261fcf1eac2a2e9288", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324016178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324016178", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "564fa89fa147d542879d4ad7d8b9564103bc572af9da1acd609a18d2804ccf21" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111217799999395, + "build_seconds": 9.285103848000006, + "workflow_seconds": null, + "total_seconds": 9.386216026, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0053126480000003085, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.386216026 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174541", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.30168808900000954, + "build_seconds": 8.787957718000001, + "workflow_seconds": null, + "total_seconds": 9.08964580700001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.628428673000002, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.08964580700001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174540", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/report.md b/results/gogs-moon/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..ae9878ec9 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-seed-02 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10111217799999395 | 9.285103848000006 | 9.386216026 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.30168808900000954 | 8.787957718000001 | 9.08964580700001 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..772c20169 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111217799999395, + "build_seconds": 9.285103848000006, + "workflow_seconds": null, + "total_seconds": 9.386216026, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0053126480000003085, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.386216026 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174541", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..a24b99b64 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.30168808900000954, + "build_seconds": 8.787957718000001, + "workflow_seconds": null, + "total_seconds": 9.08964580700001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.628428673000002, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.08964580700001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324016178", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174540", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324016178-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324016178" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-02/series.json b/results/gogs-moon/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..7caab2670 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "23b7faf2d7426940a8b1dc9065cf32fc3bc7adac6f2ce1261fcf1eac2a2e9288" +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json b/results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json new file mode 100644 index 000000000..6b08342e2 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/completions/37325135541.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325135541", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d926248648e2c843978677c49733b22262f18546233d52922b5e1e6b6c786bf2" +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..5d60f29fe --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3", + "series_id": "cadence-rolling-seed-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:29:33Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325135541, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541" +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/report.json b/results/gogs-moon/cadence-rolling-seed-03/report.json new file mode 100644 index 000000000..ec4f9896d --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "1778935d51f88873928e2fa318c7d98d3ce6a582be6afd3869dc2238b57d0aad", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325135541", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325135541", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d926248648e2c843978677c49733b22262f18546233d52922b5e1e6b6c786bf2" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010086570000226, + "build_seconds": 8.791259834000016, + "workflow_seconds": null, + "total_seconds": 8.892268491000038, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004114332999989756, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.892268491000038 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174591", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20145711899999696, + "build_seconds": 9.786454968000044, + "workflow_seconds": null, + "total_seconds": 9.98791208700004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.7367870319999952, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.98791208700004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174590", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/report.md b/results/gogs-moon/cadence-rolling-seed-03/report.md new file mode 100644 index 000000000..281f1c931 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-seed-03 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1010086570000226 | 8.791259834000016 | 8.892268491000038 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20145711899999696 | 9.786454968000044 | 9.98791208700004 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..fae91ffa7 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010086570000226, + "build_seconds": 8.791259834000016, + "workflow_seconds": null, + "total_seconds": 8.892268491000038, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004114332999989756, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.892268491000038 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174591", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d88d0e545 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20145711899999696, + "build_seconds": 9.786454968000044, + "workflow_seconds": null, + "total_seconds": 9.98791208700004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.7367870319999952, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.98791208700004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325135541", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174590", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325135541-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325135541" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed-03/series.json b/results/gogs-moon/cadence-rolling-seed-03/series.json new file mode 100644 index 000000000..d9f42ab2f --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed-03", + "created_at": "2026-10-05T14:29:29Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "0da0b8791beff9333b016060e6279157d93d1924fbab77071da92848ab541e13", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "1778935d51f88873928e2fa318c7d98d3ce6a582be6afd3869dc2238b57d0aad" +} diff --git a/results/gogs-moon/cadence-rolling-seed/completions/37322598011.json b/results/gogs-moon/cadence-rolling-seed/completions/37322598011.json new file mode 100644 index 000000000..03ab21131 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/completions/37322598011.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322598011", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322598011", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1e70ae525c9b8d7c2a0e47fb57008f595706a292bbbde0099a3862a0729a3307" +} diff --git a/results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..ac7ca3c1c --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322598011, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322598011" +} diff --git a/results/gogs-moon/cadence-rolling-seed/report.json b/results/gogs-moon/cadence-rolling-seed/report.json new file mode 100644 index 000000000..842f88978 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "1fc9fdbfd692a7c3b1c3479e4c1cc1a1f55190e7a7f4a7f781986271e4a5cee8", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322598011", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322598011", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1e70ae525c9b8d7c2a0e47fb57008f595706a292bbbde0099a3862a0729a3307" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10112419299997555, + "build_seconds": 9.287674577000018, + "workflow_seconds": null, + "total_seconds": 9.388798769999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004315487999974721, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.388798769999994 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174434", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10093500900000407, + "build_seconds": 8.487470181999996, + "workflow_seconds": null, + "total_seconds": 8.588405191, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9377041529999985, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.588405191 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174433", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/cadence-rolling-seed/report.md b/results/gogs-moon/cadence-rolling-seed/report.md new file mode 100644 index 000000000..8e1e26a25 --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# gogs-moon: cadence-rolling-seed + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10112419299997555 | 9.287674577000018 | 9.388798769999994 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.10093500900000407 | 8.487470181999996 | 8.588405191 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..8433ef3fb --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10112419299997555, + "build_seconds": 9.287674577000018, + "workflow_seconds": null, + "total_seconds": 9.388798769999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004315487999974721, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.388798769999994 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174434", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..b8d702e1d --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10093500900000407, + "build_seconds": 8.487470181999996, + "workflow_seconds": null, + "total_seconds": 8.588405191, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9377041529999985, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 8.588405191 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-rolling-cadence-20261005-gogs-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322598011", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174433", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322598011-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322598011" + ] +} diff --git a/results/gogs-moon/cadence-rolling-seed/series.json b/results/gogs-moon/cadence-rolling-seed/series.json new file mode 100644 index 000000000..493eba22b --- /dev/null +++ b/results/gogs-moon/cadence-rolling-seed/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "c6331fef820be091ea63be5d7070dd471a919b248e4e2fa903d50e4245d847a0", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-gogs-moon" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "1fc9fdbfd692a7c3b1c3479e4c1cc1a1f55190e7a7f4a7f781986271e4a5cee8" +} diff --git a/results/helix-nix/cadence-fresh-01/completions/37320867908.json b/results/helix-nix/cadence-fresh-01/completions/37320867908.json new file mode 100644 index 000000000..b5e5c792b --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/completions/37320867908.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320867908", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320867908", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e103123da0dfd86047c1780a4ea86e37c4d55786742b3fb66d000554d9172fd0" +} diff --git a/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json new file mode 100644 index 000000000..e6891e216 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark-retry.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:29:07Z", + "ref": "cadence-fresh-qualification", + "state": "requested", + "run_id": 37325075463, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325075463" +} diff --git a/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..ebe223dd2 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:28:34Z", + "ref": "shared-benchmark-cadence", + "state": "rejected", + "error": "GitHub HTTP 422: merged PR branch shared-benchmark-cadence was deleted; no ref found." +} diff --git a/results/helix-nix/cadence-fresh-01/report.json b/results/helix-nix/cadence-fresh-01/report.json new file mode 100644 index 000000000..71a10214c --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/report.json @@ -0,0 +1,801 @@ +{ + "schema_version": 1, + "plan_sha256": "c2a0ba9332802d28a66f0938b4fd31a43ff928f6d8e7348f2f07f37e1c933421", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320867908", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320867908", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e103123da0dfd86047c1780a4ea86e37c4d55786742b3fb66d000554d9172fd0" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 2, + "cold", + "boringcache" + ], + [ + 2, + "cold", + "cachix" + ], + [ + 2, + "warm", + "boringcache" + ], + [ + 2, + "warm", + "cachix" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 219.9609663066821, + "min": 219.9609663066821, + "max": 219.9609663066821 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 6.481956831083494, + "min": 6.481956831083494, + "max": 6.481956831083494 + }, + "storage": { + "median": 46171613, + "min": 46171613, + "max": 46171613 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 363.82386535294876, + "min": 363.82386535294876, + "max": 363.82386535294876 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 3.6484187521867284, + "min": 3.6484187521867284, + "max": 3.6484187521867284 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.364945411682129, + "build_seconds": 215.59602089499998, + "workflow_seconds": null, + "total_seconds": 219.9609663066821, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 219.9609663066821 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174370", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.811720848083496, + "build_seconds": 0.6702359829999978, + "workflow_seconds": null, + "total_seconds": 6.481956831083494, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.481956831083494 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174502", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4744367599487305, + "build_seconds": 361.349428593, + "workflow_seconds": null, + "total_seconds": 363.82386535294876, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 363.82386535294876 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174369", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3420071601867676, + "build_seconds": 1.3064115919999608, + "workflow_seconds": null, + "total_seconds": 3.6484187521867284, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.6484187521867284 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174503", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/cadence-fresh-01/report.md b/results/helix-nix/cadence-fresh-01/report.md new file mode 100644 index 000000000..67b771b52 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/report.md @@ -0,0 +1,36 @@ +# helix-nix: cadence-fresh-01 + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 4/8 recorded; 0 failed; 4 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 1 | 219.9609663066821 | 219.9609663066821–219.9609663066821 | unmeasured | 0 | +| Warm build | BoringCache | 1 | 6.481956831083494 | 6.481956831083494–6.481956831083494 | 46171613 | 1 | +| Cold build | Cachix | 1 | 363.82386535294876 | 363.82386535294876–363.82386535294876 | unmeasured | 0 | +| Warm build | Cachix | 1 | 3.6484187521867284 | 3.6484187521867284–3.6484187521867284 | unmeasured | 0 | + +Missing observations: + +- Sample 2, BoringCache, Cold build +- Sample 2, Cachix, Cold build +- Sample 2, BoringCache, Warm build +- Sample 2, Cachix, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 4.364945411682129 | 215.59602089499998 | 219.9609663066821 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 5.811720848083496 | 0.6702359829999978 | 6.481956831083494 | 46171613 | boringcache-check | hit | [JSON](runs/1-boringcache-warm.json) | +| 1 | Cachix | Cold build | 2.4744367599487305 | 361.349428593 | 363.82386535294876 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | +| 1 | Cachix | Warm build | 2.3420071601867676 | 1.3064115919999608 | 3.6484187521867284 | unmeasured | unmeasured | hit | [JSON](runs/1-cachix-warm.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..fdb35e09a --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.364945411682129, + "build_seconds": 215.59602089499998, + "workflow_seconds": null, + "total_seconds": 219.9609663066821, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 219.9609663066821 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174370", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..aa31a95d6 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.811720848083496, + "build_seconds": 0.6702359829999978, + "workflow_seconds": null, + "total_seconds": 6.481956831083494, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.481956831083494 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s1-r37320867908-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174502", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json b/results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json new file mode 100644 index 000000000..3a47e8d5e --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4744367599487305, + "build_seconds": 361.349428593, + "workflow_seconds": null, + "total_seconds": 363.82386535294876, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 363.82386535294876 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174369", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json b/results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json new file mode 100644 index 000000000..42fc88da7 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/1-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3420071601867676, + "build_seconds": 1.3064115919999608, + "workflow_seconds": null, + "total_seconds": 3.6484187521867284, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.6484187521867284 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320867908", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174503", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320867908-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320867908" + ] +} diff --git a/results/helix-nix/cadence-rolling-advance/completions/37323117664.json b/results/helix-nix/cadence-rolling-advance/completions/37323117664.json new file mode 100644 index 000000000..bce8685e9 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/completions/37323117664.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323117664", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ccb6af059c8aae587960411129f7d4c7908c4555fe54f657899e18aa3e2b060" +} diff --git a/results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json b/results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..8785394b9 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:32Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323117664, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664" +} diff --git a/results/helix-nix/cadence-rolling-advance/report.json b/results/helix-nix/cadence-rolling-advance/report.json new file mode 100644 index 000000000..df30f8632 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/report.json @@ -0,0 +1,412 @@ +{ + "schema_version": 1, + "plan_sha256": "c1e34a971796d860ed2a472ecb705e2ee919c904b86ae87e3239d2dc8422241a", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323117664", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323117664", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ccb6af059c8aae587960411129f7d4c7908c4555fe54f657899e18aa3e2b060" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 295.80389107529044, + "min": 295.80389107529044, + "max": 295.80389107529044 + }, + "storage": { + "median": 46171799, + "min": 46171799, + "max": 46171799 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 3.877758286099265, + "min": 3.877758286099265, + "max": 3.877758286099265 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6.215389013290405, + "build_seconds": 289.58850206200003, + "workflow_seconds": null, + "total_seconds": 295.80389107529044, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 295.80389107529044 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171799, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": true, + "measured_bytes": 46171799, + "unmeasured_tags": [], + "total_bytes": 46171799, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171799, + "compressed_size": 46171799, + "size": 46171799 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174619", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.420297861099243, + "build_seconds": 1.457460425000022, + "workflow_seconds": null, + "total_seconds": 3.877758286099265, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.877758286099265 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174620", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/cadence-rolling-advance/report.md b/results/helix-nix/cadence-rolling-advance/report.md new file mode 100644 index 000000000..42f482148 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/report.md @@ -0,0 +1,29 @@ +# helix-nix: cadence-rolling-advance + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 295.80389107529044 | 295.80389107529044–295.80389107529044 | 46171799 | 1 | +| Changed-source build | Cachix | 1 | 3.877758286099265 | 3.877758286099265–3.877758286099265 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 6.215389013290405 | 289.58850206200003 | 295.80389107529044 | 46171799 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.420297861099243 | 1.457460425000022 | 3.877758286099265 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..e0f483411 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6.215389013290405, + "build_seconds": 289.58850206200003, + "workflow_seconds": null, + "total_seconds": 295.80389107529044, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 295.80389107529044 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171799, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": true, + "measured_bytes": 46171799, + "unmeasured_tags": [], + "total_bytes": 46171799, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171799, + "compressed_size": 46171799, + "size": 46171799 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174619", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] +} diff --git a/results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json b/results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json new file mode 100644 index 000000000..bce549ccc --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.420297861099243, + "build_seconds": 1.457460425000022, + "workflow_seconds": null, + "total_seconds": 3.877758286099265, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.877758286099265 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323117664", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174620", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323117664-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323117664" + ] +} diff --git a/results/helix-nix/cadence-rolling-advance/series.json b/results/helix-nix/cadence-rolling-advance/series.json new file mode 100644 index 000000000..ef44c63d8 --- /dev/null +++ b/results/helix-nix/cadence-rolling-advance/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "4d239533a6f8812605515a2af39709029185fff23d50f253cb0d5351f8c3fe2c", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "c1e34a971796d860ed2a472ecb705e2ee919c904b86ae87e3239d2dc8422241a" +} diff --git a/results/helix-nix/cadence-rolling-seed/completions/37322604229.json b/results/helix-nix/cadence-rolling-seed/completions/37322604229.json new file mode 100644 index 000000000..2c1533c2a --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/completions/37322604229.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322604229", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3f29172cdc9625dc804e52f16b92615b974db093eb00db3544ec28280cae465d" +} diff --git a/results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json b/results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..5268e6813 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322604229, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229" +} diff --git a/results/helix-nix/cadence-rolling-seed/report.json b/results/helix-nix/cadence-rolling-seed/report.json new file mode 100644 index 000000000..34d207f7b --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/report.json @@ -0,0 +1,407 @@ +{ + "schema_version": 1, + "plan_sha256": "76029a49e4208bb2c349ee7fdbca054c98b21b7afe37a050f9616d675d20002b", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322604229", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322604229", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3f29172cdc9625dc804e52f16b92615b974db093eb00db3544ec28280cae465d" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 273.0119839951974, + "min": 273.0119839951974, + "max": 273.0119839951974 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 318.47876023020336, + "min": 318.47876023020336, + "max": 318.47876023020336 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1.9675590991973877, + "build_seconds": 271.044424896, + "workflow_seconds": null, + "total_seconds": 273.0119839951974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 273.0119839951974 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174509", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.734213352203369, + "build_seconds": 315.744546878, + "workflow_seconds": null, + "total_seconds": 318.47876023020336, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 318.47876023020336 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174508", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/cadence-rolling-seed/report.md b/results/helix-nix/cadence-rolling-seed/report.md new file mode 100644 index 000000000..9361ebcbb --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/report.md @@ -0,0 +1,29 @@ +# helix-nix: cadence-rolling-seed + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 273.0119839951974 | 273.0119839951974–273.0119839951974 | unmeasured | 0 | +| Changed-source build | Cachix | 1 | 318.47876023020336 | 318.47876023020336–318.47876023020336 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 1.9675590991973877 | 271.044424896 | 273.0119839951974 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.734213352203369 | 315.744546878 | 318.47876023020336 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..b4e844734 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1.9675590991973877, + "build_seconds": 271.044424896, + "workflow_seconds": null, + "total_seconds": 273.0119839951974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 273.0119839951974 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "tag": "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-rolling-cadence-20261005-helix-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174509", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] +} diff --git a/results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json b/results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json new file mode 100644 index 000000000..13ff6dfc9 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.734213352203369, + "build_seconds": 315.744546878, + "workflow_seconds": null, + "total_seconds": 318.47876023020336, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 318.47876023020336 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322604229", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174508", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322604229-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322604229" + ] +} diff --git a/results/helix-nix/cadence-rolling-seed/series.json b/results/helix-nix/cadence-rolling-seed/series.json new file mode 100644 index 000000000..1b9262b32 --- /dev/null +++ b/results/helix-nix/cadence-rolling-seed/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "84c15c7ae5348fe94761514bf96d1cdc98ac24bea6fa3dd0ebd1e6571d10946c", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-helix-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e0842e6aaebf53a317fcb686b08b2ac2413d251e" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "76029a49e4208bb2c349ee7fdbca054c98b21b7afe37a050f9616d675d20002b" +} diff --git a/results/immich/cadence-rolling-advance/completions/37323121773.json b/results/immich/cadence-rolling-advance/completions/37323121773.json new file mode 100644 index 000000000..c66ea5ef4 --- /dev/null +++ b/results/immich/cadence-rolling-advance/completions/37323121773.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323121773", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f57898a2ccdc269f94623b46432c6b6280bfd6ef88b9288f990b9d630d852949" +} diff --git a/results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json b/results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..4471eb729 --- /dev/null +++ b/results/immich/cadence-rolling-advance/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "immich", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:34Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323121773, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773" +} diff --git a/results/immich/cadence-rolling-advance/report.json b/results/immich/cadence-rolling-advance/report.json new file mode 100644 index 000000000..cac73388f --- /dev/null +++ b/results/immich/cadence-rolling-advance/report.json @@ -0,0 +1,445 @@ +{ + "schema_version": 1, + "plan_sha256": "02346d5e27c00699baa7c1376b045c7d89d732f05546d65a5be05de961ccb01a", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323121773", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323121773", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f57898a2ccdc269f94623b46432c6b6280bfd6ef88b9288f990b9d630d852949" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 333, + "min": 333, + "max": 333 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 180, + "min": 180, + "max": 180 + }, + "storage": { + "median": 1750703493, + "min": 1750703493, + "max": 1750703493 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 328, + "workflow_seconds": 339, + "total_seconds": 333, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 333 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174468", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 185, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1750703493, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1750703493, + "unmeasured_tags": [], + "total_bytes": 1750703493, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "450988c6-cca4-460a-9667-b6274cc4970f", + "manifest_root_digest": "sha256:98e22c4f5630762cfa17505061947d0c6e87e52533bc629501c94922aebb3dfb", + "kv_total_size": 0, + "compressed_size": 1750703493, + "size": 1750703493 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174467", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] + } + ], + "exclusions": [] +} diff --git a/results/immich/cadence-rolling-advance/report.md b/results/immich/cadence-rolling-advance/report.md new file mode 100644 index 000000000..3082d959e --- /dev/null +++ b/results/immich/cadence-rolling-advance/report.md @@ -0,0 +1,29 @@ +# immich: cadence-rolling-advance + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 333 | 333–333 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 180 | 180–180 | 1750703493 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 328 | 333 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 180 | 180 | 1750703493 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json b/results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..151f7ee74 --- /dev/null +++ b/results/immich/cadence-rolling-advance/runs/1-actions-cache-commit.json @@ -0,0 +1,167 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 328, + "workflow_seconds": 339, + "total_seconds": 333, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 333 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174468", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] +} diff --git a/results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..9571a4fb6 --- /dev/null +++ b/results/immich/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,213 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 185, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1750703493, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1750703493, + "unmeasured_tags": [], + "total_bytes": 1750703493, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "450988c6-cca4-460a-9667-b6274cc4970f", + "manifest_root_digest": "sha256:98e22c4f5630762cfa17505061947d0c6e87e52533bc629501c94922aebb3dfb", + "kv_total_size": 0, + "compressed_size": 1750703493, + "size": 1750703493 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "0521a662dd4c135fc311fd1803d33756971af9c5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323121773", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174467", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323121773-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323121773" + ] +} diff --git a/results/immich/cadence-rolling-advance/series.json b/results/immich/cadence-rolling-advance/series.json new file mode 100644 index 000000000..d82c43eed --- /dev/null +++ b/results/immich/cadence-rolling-advance/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "ff55ded680c596c21c9ef05fda213d5d6be723b2f445533e6f97f2a1b5567199", + "variant": null, + "workflow_inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "0521a662dd4c135fc311fd1803d33756971af9c5", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "02346d5e27c00699baa7c1376b045c7d89d732f05546d65a5be05de961ccb01a" +} diff --git a/results/immich/cadence-rolling-seed/completions/37321739971.json b/results/immich/cadence-rolling-seed/completions/37321739971.json new file mode 100644 index 000000000..85c510141 --- /dev/null +++ b/results/immich/cadence-rolling-seed/completions/37321739971.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37321739971", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "07eb4d0fc8d5073ad36895d85586e3b25a17b6d8671c966903a76bacd3f7f27d" +} diff --git a/results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json b/results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0b0385ad3 --- /dev/null +++ b/results/immich/cadence-rolling-seed/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "immich", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:04:15Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321739971, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971" +} diff --git a/results/immich/cadence-rolling-seed/report.json b/results/immich/cadence-rolling-seed/report.json new file mode 100644 index 000000000..c256a1de7 --- /dev/null +++ b/results/immich/cadence-rolling-seed/report.json @@ -0,0 +1,445 @@ +{ + "schema_version": 1, + "plan_sha256": "18841129bbf62c7426a8451343792d2b8276bb379fc46fc6666e01f6666b7817", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37321739971", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321739971", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "07eb4d0fc8d5073ad36895d85586e3b25a17b6d8671c966903a76bacd3f7f27d" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 364, + "min": 364, + "max": 364 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 187, + "min": 187, + "max": 187 + }, + "storage": { + "median": 1756603878, + "min": 1756603878, + "max": 1756603878 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 352, + "workflow_seconds": 370, + "total_seconds": 364, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 364 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174347", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 187, + "workflow_seconds": 194, + "total_seconds": 187, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 187 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1756603878, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1756603878, + "unmeasured_tags": [], + "total_bytes": 1756603878, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "30ebce4e-787e-4a47-8739-c431adc97c86", + "manifest_root_digest": "sha256:1f8ec2caa2839792df37e0036ea0a297c1df98fac25d22daa13b5221b97ce453", + "kv_total_size": 0, + "compressed_size": 1756603878, + "size": 1756603878 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174348", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] + } + ], + "exclusions": [] +} diff --git a/results/immich/cadence-rolling-seed/report.md b/results/immich/cadence-rolling-seed/report.md new file mode 100644 index 000000000..55eea9257 --- /dev/null +++ b/results/immich/cadence-rolling-seed/report.md @@ -0,0 +1,29 @@ +# immich: cadence-rolling-seed + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 364 | 364–364 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 187 | 187–187 | 1756603878 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 12 | 352 | 364 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 187 | 187 | 1756603878 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json b/results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..d25926bd6 --- /dev/null +++ b/results/immich/cadence-rolling-seed/runs/1-actions-cache-commit.json @@ -0,0 +1,167 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 352, + "workflow_seconds": 370, + "total_seconds": 364, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 364 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-rolling-cadence-20261005-immich", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174347", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] +} diff --git a/results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d4a6e2efc --- /dev/null +++ b/results/immich/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,213 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": { + "load_image": "true" + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 187, + "workflow_seconds": 194, + "total_seconds": 187, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 187 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "immich-server-rolling-cadence-20261005-immich-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + }, + "tag": "immich-server-rolling-cadence-20261005-immich-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1756603878, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1756603878, + "unmeasured_tags": [], + "total_bytes": 1756603878, + "observations": [ + { + "requested_tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "tag": "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "30ebce4e-787e-4a47-8739-c431adc97c86", + "manifest_root_digest": "sha256:1f8ec2caa2839792df37e0036ea0a297c1df98fac25d22daa13b5221b97ce453", + "kv_total_size": 0, + "compressed_size": 1756603878, + "size": 1756603878 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-rolling-cadence-20261005-immich-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321739971", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174348", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321739971-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321739971" + ] +} diff --git a/results/immich/cadence-rolling-seed/series.json b/results/immich/cadence-rolling-seed/series.json new file mode 100644 index 000000000..5d799c1c1 --- /dev/null +++ b/results/immich/cadence-rolling-seed/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "3cb0f35d5beeb62d69b86623d59daf1a2c08229910a83b3feb5c126e295cb1dc", + "variant": null, + "workflow_inputs": { + "case_id": "immich", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-immich" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "18841129bbf62c7426a8451343792d2b8276bb379fc46fc6666e01f6666b7817" +} diff --git a/results/mastodon/cadence-rolling-advance-server/completions/37323126507.json b/results/mastodon/cadence-rolling-advance-server/completions/37323126507.json new file mode 100644 index 000000000..6ad771b1a --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/completions/37323126507.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-server", + "sample": 1, + "run_id": "37323126507", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c7cdaec26745cefee9e346dba7b3ca2260869793a5a43790bb7c2919576e8bc" +} diff --git a/results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0c5f3a0cf --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server", + "series_id": "cadence-rolling-advance-server", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:36Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323126507, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507" +} diff --git a/results/mastodon/cadence-rolling-advance-server/report.json b/results/mastodon/cadence-rolling-advance-server/report.json new file mode 100644 index 000000000..721a2fdfb --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "94d5db9803d90b2f237a3d998a5c6cf43cbd12ad3cdb27c6ff8c897e43299d40", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-server", + "sample": 1, + "run_id": "37323126507", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323126507", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c7cdaec26745cefee9e346dba7b3ca2260869793a5a43790bb7c2919576e8bc" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 299, + "min": 299, + "max": 299 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 180, + "min": 180, + "max": 180 + }, + "storage": { + "median": 1131047375, + "min": 1131047375, + "max": 1131047375 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 293, + "workflow_seconds": 308, + "total_seconds": 299, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 299 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174472", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 187, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1131047375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1131047375, + "unmeasured_tags": [], + "total_bytes": 1131047375, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "bb8a82c2-185a-45ab-a182-59a2bab44ac2", + "manifest_root_digest": "sha256:f39b48d505b211e9e0a5be7430c8e0484cb1677ca5f39c6039c81897dcff5176", + "kv_total_size": 0, + "compressed_size": 1131047375, + "size": 1131047375 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174471", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-advance-server/report.md b/results/mastodon/cadence-rolling-advance-server/report.md new file mode 100644 index 000000000..c02aa05c3 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-advance-server + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 299 | 299–299 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 180 | 180–180 | 1131047375 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 6 | 293 | 299 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 180 | 180 | 1131047375 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..310464ca2 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 293, + "workflow_seconds": 308, + "total_seconds": 299, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 299 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174472", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json new file mode 100644 index 000000000..63f2fa0fa --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 180, + "workflow_seconds": 187, + "total_seconds": 180, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 180 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1131047375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1131047375, + "unmeasured_tags": [], + "total_bytes": 1131047375, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "bb8a82c2-185a-45ab-a182-59a2bab44ac2", + "manifest_root_digest": "sha256:f39b48d505b211e9e0a5be7430c8e0484cb1677ca5f39c6039c81897dcff5176", + "kv_total_size": 0, + "compressed_size": 1131047375, + "size": 1131047375 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323126507", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174471", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323126507-1", + "series": { + "id": "cadence-rolling-advance-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323126507" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-server/series.json b/results/mastodon/cadence-rolling-advance-server/series.json new file mode 100644 index 000000000..a86c5f335 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-server/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-server", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "variant": "server", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "94d5db9803d90b2f237a3d998a5c6cf43cbd12ad3cdb27c6ff8c897e43299d40" +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json b/results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json new file mode 100644 index 000000000..e21bd88d8 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/completions/37323131290.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-streaming", + "sample": 1, + "run_id": "37323131290", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "acb6f806ef1e53607119c0ef8efcd7512dcd83ba7ed89c2b9940e5cacaf32884" +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..cadc4eef7 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming", + "series_id": "cadence-rolling-advance-streaming", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:39Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323131290, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290" +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/report.json b/results/mastodon/cadence-rolling-advance-streaming/report.json new file mode 100644 index 000000000..68fc7d95a --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "a5828cd7050d1f29ba9eecff83d2661ac333582448b631d428141d83d99b16f0", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-streaming", + "sample": 1, + "run_id": "37323131290", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323131290", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "acb6f806ef1e53607119c0ef8efcd7512dcd83ba7ed89c2b9940e5cacaf32884" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 44, + "min": 44, + "max": 44 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 23, + "min": 23, + "max": 23 + }, + "storage": { + "median": 106316055, + "min": 106316055, + "max": 106316055 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 21, + "build_seconds": 23, + "workflow_seconds": 48, + "total_seconds": 44, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 44 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174473", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 23, + "workflow_seconds": 29, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106316055, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106316055, + "unmeasured_tags": [], + "total_bytes": 106316055, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "08c60a15-9c1d-4d96-a20a-3466790ea1b2", + "manifest_root_digest": "sha256:d7e2d159bd05e94b274c139460296fb261ed19046a9f321f6b07b4570ff99cc6", + "kv_total_size": 0, + "compressed_size": 106316055, + "size": 106316055 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174474", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/report.md b/results/mastodon/cadence-rolling-advance-streaming/report.md new file mode 100644 index 000000000..779eb1b65 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-advance-streaming + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 44 | 44–44 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 23 | 23–23 | 106316055 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 21 | 23 | 44 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 23 | 23 | 106316055 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..a6fdf6583 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 21, + "build_seconds": 23, + "workflow_seconds": 48, + "total_seconds": 44, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 44 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174473", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d7ace5173 --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 23, + "workflow_seconds": 29, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106316055, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106316055, + "unmeasured_tags": [], + "total_bytes": 106316055, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "08c60a15-9c1d-4d96-a20a-3466790ea1b2", + "manifest_root_digest": "sha256:d7e2d159bd05e94b274c139460296fb261ed19046a9f321f6b07b4570ff99cc6", + "kv_total_size": 0, + "compressed_size": 106316055, + "size": 106316055 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "842b86096dbe3067a02e3f16809e2368eaeddce9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323131290", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174474", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323131290-1", + "series": { + "id": "cadence-rolling-advance-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323131290" + ] +} diff --git a/results/mastodon/cadence-rolling-advance-streaming/series.json b/results/mastodon/cadence-rolling-advance-streaming/series.json new file mode 100644 index 000000000..dbc01391c --- /dev/null +++ b/results/mastodon/cadence-rolling-advance-streaming/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-advance-streaming", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "24b1333197dc2d40e54cb227b05b2cc05482fe3722059147ae4f403d3f513277", + "variant": "streaming", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "842b86096dbe3067a02e3f16809e2368eaeddce9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "a5828cd7050d1f29ba9eecff83d2661ac333582448b631d428141d83d99b16f0" +} diff --git a/results/mastodon/cadence-rolling-seed-server/completions/37321744434.json b/results/mastodon/cadence-rolling-seed-server/completions/37321744434.json new file mode 100644 index 000000000..e4b3d5fbe --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/completions/37321744434.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-server", + "sample": 1, + "run_id": "37321744434", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a548877f8cf73415bd6153d723a27a1d1c008a24881391acb25ac4a8338c763a" +} diff --git a/results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..d54dd94ed --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server", + "series_id": "cadence-rolling-seed-server", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:18Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321744434, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434" +} diff --git a/results/mastodon/cadence-rolling-seed-server/report.json b/results/mastodon/cadence-rolling-seed-server/report.json new file mode 100644 index 000000000..ec570194a --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "561a3c72c845a65485bb0a53dfd0e2be0baa59ac65533a44505140b8fac8d7c6", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-server", + "sample": 1, + "run_id": "37321744434", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321744434", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a548877f8cf73415bd6153d723a27a1d1c008a24881391acb25ac4a8338c763a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 493, + "min": 493, + "max": 493 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 529, + "min": 529, + "max": 529 + }, + "storage": { + "median": 1130441371, + "min": 1130441371, + "max": 1130441371 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 484, + "workflow_seconds": 500, + "total_seconds": 493, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 493 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174351", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 529, + "workflow_seconds": 538, + "total_seconds": 529, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 529 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130441371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130441371, + "unmeasured_tags": [], + "total_bytes": 1130441371, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "10417425-d059-41fe-81d0-c0f00e9a00ac", + "manifest_root_digest": "sha256:09d1b510600ce403588247bd81f36109c76272d0192fe149c873aa723961e238", + "kv_total_size": 0, + "compressed_size": 1130441371, + "size": 1130441371 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174352", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-seed-server/report.md b/results/mastodon/cadence-rolling-seed-server/report.md new file mode 100644 index 000000000..39c072572 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-seed-server + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 493 | 493–493 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 529 | 529–529 | 1130441371 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 9 | 484 | 493 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 529 | 529 | 1130441371 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..6f2426b2f --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 484, + "workflow_seconds": 500, + "total_seconds": 493, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 493 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-rolling-cadence-20261005-mastodon-server", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174351", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json new file mode 100644 index 000000000..62b22dafc --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 529, + "workflow_seconds": 538, + "total_seconds": 529, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 529 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "mastodon-rolling-cadence-20261005-mastodon-server-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + }, + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130441371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130441371, + "unmeasured_tags": [], + "total_bytes": 1130441371, + "observations": [ + { + "requested_tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "tag": "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "10417425-d059-41fe-81d0-c0f00e9a00ac", + "manifest_root_digest": "sha256:09d1b510600ce403588247bd81f36109c76272d0192fe149c873aa723961e238", + "kv_total_size": 0, + "compressed_size": 1130441371, + "size": 1130441371 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-rolling-cadence-20261005-mastodon-server-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321744434", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174352", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321744434-1", + "series": { + "id": "cadence-rolling-seed-server", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321744434" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-server/series.json b/results/mastodon/cadence-rolling-seed-server/series.json new file mode 100644 index 000000000..d443f32f6 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-server/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-server", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "variant": "server", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-server", + "variant": "server" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "561a3c72c845a65485bb0a53dfd0e2be0baa59ac65533a44505140b8fac8d7c6" +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json b/results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json new file mode 100644 index 000000000..46f92f822 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/completions/37321748690.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-streaming", + "sample": 1, + "run_id": "37321748690", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a1f7eca3d823149f094a2316b6c4310d255e6f71752359e90b344b8e55ed88c" +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json b/results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..50eaa1e3a --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming", + "series_id": "cadence-rolling-seed-streaming", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:20Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321748690, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690" +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/report.json b/results/mastodon/cadence-rolling-seed-streaming/report.json new file mode 100644 index 000000000..203e0e399 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/report.json @@ -0,0 +1,525 @@ +{ + "schema_version": 1, + "plan_sha256": "f673553fdf484d6f3aadc8ecaaa224a32e1985da51fbe20076919e043bb5e6ca", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-streaming", + "sample": 1, + "run_id": "37321748690", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321748690", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a1f7eca3d823149f094a2316b6c4310d255e6f71752359e90b344b8e55ed88c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 32, + "min": 32, + "max": 32 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 27, + "min": 27, + "max": 27 + }, + "storage": { + "median": 106284886, + "min": 106284886, + "max": 106284886 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 24, + "workflow_seconds": 39, + "total_seconds": 32, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 32 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174354", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 27, + "workflow_seconds": 35, + "total_seconds": 27, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 27 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106284886, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106284886, + "unmeasured_tags": [], + "total_bytes": 106284886, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eac04b9-8b45-49eb-9791-3d3296be20b7", + "manifest_root_digest": "sha256:57a408410d24dfd969eef8ffd6dbf28ec28a8099c8cf6d471558a98a333ad2aa", + "kv_total_size": 0, + "compressed_size": 106284886, + "size": 106284886 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174355", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/report.md b/results/mastodon/cadence-rolling-seed-streaming/report.md new file mode 100644 index 000000000..d23f27b4e --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/report.md @@ -0,0 +1,29 @@ +# mastodon: cadence-rolling-seed-streaming + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 32 | 32–32 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 27 | 27–27 | 106284886 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 8 | 24 | 32 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 27 | 27 | 106284886 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json b/results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..f1ef2cc19 --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/runs/1-actions-cache-commit.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 24, + "workflow_seconds": 39, + "total_seconds": 32, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 32 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174354", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json b/results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json new file mode 100644 index 000000000..332bc30da --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/runs/1-boringcache-commit.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server", + "load_image": "true" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 27, + "workflow_seconds": 35, + "total_seconds": 27, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 27 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + }, + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106284886, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106284886, + "unmeasured_tags": [], + "total_bytes": 106284886, + "observations": [ + { + "requested_tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "tag": "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eac04b9-8b45-49eb-9791-3d3296be20b7", + "manifest_root_digest": "sha256:57a408410d24dfd969eef8ffd6dbf28ec28a8099c8cf6d471558a98a333ad2aa", + "kv_total_size": 0, + "compressed_size": 106284886, + "size": 106284886 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-rolling-cadence-20261005-mastodon-streaming-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321748690", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174355", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321748690-1", + "series": { + "id": "cadence-rolling-seed-streaming", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321748690" + ] +} diff --git a/results/mastodon/cadence-rolling-seed-streaming/series.json b/results/mastodon/cadence-rolling-seed-streaming/series.json new file mode 100644 index 000000000..6a532306d --- /dev/null +++ b/results/mastodon/cadence-rolling-seed-streaming/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "cadence-rolling-seed-streaming", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "23afcc549fc2901c847fff96c731ac460952e291cb513c2a46148ded4206ad68", + "variant": "streaming", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-mastodon-streaming", + "variant": "streaming" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "f673553fdf484d6f3aadc8ecaaa224a32e1985da51fbe20076919e043bb5e6ca" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json b/results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json new file mode 100644 index 000000000..93c7c9a52 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/completions/37324214890.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324214890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a49c2ba2066ca33e2fef110f6cf68bbff938c9ba3212571c8dc6aaf97a99799d" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..fc0377830 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:43Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324214890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/report.json b/results/msgpack-sbt/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..88f52275a --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "87754745e5a804e3a837535e69b5636562773f538f358d17eb5631502268ac20", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324214890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324214890", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a49c2ba2066ca33e2fef110f6cf68bbff938c9ba3212571c8dc6aaf97a99799d" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10113066600000309, + "build_seconds": 22.980055022000002, + "workflow_seconds": null, + "total_seconds": 23.081185688000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013612939999916307, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 23.081185688000005 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174559", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20128656899998987, + "build_seconds": 25.342844542999984, + "workflow_seconds": null, + "total_seconds": 25.544131111999974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7568644210000173, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.544131111999974 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174560", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/report.md b/results/msgpack-sbt/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..b584a9f04 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/report.md @@ -0,0 +1,30 @@ +# msgpack-sbt: cadence-rolling-advance-02 + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10113066600000309 | 22.980055022000002 | 23.081185688000005 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20128656899998987 | 25.342844542999984 | 25.544131111999974 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..19cd9c3de --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10113066600000309, + "build_seconds": 22.980055022000002, + "workflow_seconds": null, + "total_seconds": 23.081185688000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013612939999916307, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 23.081185688000005 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174559", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..850147393 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20128656899998987, + "build_seconds": 25.342844542999984, + "workflow_seconds": null, + "total_seconds": 25.544131111999974, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7568644210000173, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.544131111999974 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324214890", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174560", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324214890-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324214890" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance-02/series.json b/results/msgpack-sbt/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..ee288bf74 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "166e0bca6dfd8bcdf9fdd75c0f612e531551c2bfa1d4bde9a9a7a596799d4822", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "87754745e5a804e3a837535e69b5636562773f538f358d17eb5631502268ac20" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json b/results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json new file mode 100644 index 000000000..154163913 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/completions/37323136088.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323136088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote msgpack-sbt commit: failure", + "msgpack-sbt rolling report: failure" + ], + "evidence_manifest_sha256": "9efdf7335b4808dd3a3bb626666cc167473699164c141d035c288bda6127701f" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..b199e1c8d --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323136088, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088" +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/report.json b/results/msgpack-sbt/cadence-rolling-advance/report.json new file mode 100644 index 000000000..12b36cf1a --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/report.json @@ -0,0 +1,223 @@ +{ + "schema_version": 1, + "plan_sha256": "cd2354c80ad3f6fa4d0543772b2ced703b776abc0cdadab4a02aa9d4889483e6", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323136088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote msgpack-sbt commit: failure", + "msgpack-sbt rolling report: failure" + ], + "evidence_manifest_sha256": "9efdf7335b4808dd3a3bb626666cc167473699164c141d035c288bda6127701f" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323136088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323136088", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote msgpack-sbt commit: failure", + "msgpack-sbt rolling report: failure" + ], + "evidence_manifest_sha256": "9efdf7335b4808dd3a3bb626666cc167473699164c141d035c288bda6127701f" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "80307420630483e2c47545a5deeca56f262b5772eb594db79cde6c03b2c477b2", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10089654600000131, + "build_seconds": 24.936086543999977, + "workflow_seconds": null, + "total_seconds": 25.03698308999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3992640499999993, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.03698308999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323136088", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174481", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323136088-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323136088" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/report.md b/results/msgpack-sbt/cadence-rolling-advance/report.md new file mode 100644 index 000000000..89c70881b --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# msgpack-sbt: cadence-rolling-advance + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323136088 failed completion checks: + +- Workflow concluded failure +- bazel-remote msgpack-sbt commit: failure +- msgpack-sbt rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.10089654600000131 | 24.936086543999977 | 25.03698308999998 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6550e89e7 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "80307420630483e2c47545a5deeca56f262b5772eb594db79cde6c03b2c477b2", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10089654600000131, + "build_seconds": 24.936086543999977, + "workflow_seconds": null, + "total_seconds": 25.03698308999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3992640499999993, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 25.03698308999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323136088", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174481", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323136088-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323136088" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-advance/series.json b/results/msgpack-sbt/cadence-rolling-advance/series.json new file mode 100644 index 000000000..2c1a05853 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-advance/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "80307420630483e2c47545a5deeca56f262b5772eb594db79cde6c03b2c477b2", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "pins": [ + { + "path": "proposal.md", + "revision": "ec75ffc01364adc6ea9b5f4d9129fd63602e6a45", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "cd2354c80ad3f6fa4d0543772b2ced703b776abc0cdadab4a02aa9d4889483e6" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json b/results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json new file mode 100644 index 000000000..baacb0813 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/completions/37324036688.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324036688", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3171de3a2ed83c157c5fd6780148832cff76ae26a9dca46d962c45e13fb00020" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..8fe011d54 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:24Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324036688, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/report.json b/results/msgpack-sbt/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..1e2756f84 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "b1568aacb3510adfb902817b524a2ea72592c56a4953b7d78320f182d14fdb98", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324036688", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324036688", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3171de3a2ed83c157c5fd6780148832cff76ae26a9dca46d962c45e13fb00020" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108540399998844, + "build_seconds": 38.53932476999999, + "workflow_seconds": null, + "total_seconds": 38.64041017399998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016968049999945833, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.64041017399998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174535", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10086146499999415, + "build_seconds": 27.882007873000006, + "workflow_seconds": null, + "total_seconds": 27.982869338, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4697909409999994, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.982869338 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174536", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/report.md b/results/msgpack-sbt/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..c9473200e --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# msgpack-sbt: cadence-rolling-seed-02 + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10108540399998844 | 38.53932476999999 | 38.64041017399998 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.10086146499999415 | 27.882007873000006 | 27.982869338 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..568f1136c --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108540399998844, + "build_seconds": 38.53932476999999, + "workflow_seconds": null, + "total_seconds": 38.64041017399998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016968049999945833, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.64041017399998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174535", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..aeefd9264 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10086146499999415, + "build_seconds": 27.882007873000006, + "workflow_seconds": null, + "total_seconds": 27.982869338, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4697909409999994, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.982869338 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324036688", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174536", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324036688-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324036688" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed-02/series.json b/results/msgpack-sbt/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..82aef726c --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed-02/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:12Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "c670b733ecd53877d91da12de19b4d8fe295a03d6477e70a5e168709e72ff371", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt-v2" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b1568aacb3510adfb902817b524a2ea72592c56a4953b7d78320f182d14fdb98" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json b/results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json new file mode 100644 index 000000000..09b003459 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/completions/37322609279.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322609279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322609279", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ee073a8869a31c3560df15b0cde8acc75ddd48d2efffadc707b7ba8da2e49fd7" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..fac4759d9 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:43Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322609279, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322609279" +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/report.json b/results/msgpack-sbt/cadence-rolling-seed/report.json new file mode 100644 index 000000000..67640dbeb --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/report.json @@ -0,0 +1,350 @@ +{ + "schema_version": 1, + "plan_sha256": "e563c124fae3f2f6a44f5c7921ea7dfc79ded103d502dd402e643463bab6cde7", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322609279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322609279", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ee073a8869a31c3560df15b0cde8acc75ddd48d2efffadc707b7ba8da2e49fd7" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111915200000965, + "build_seconds": 39.54074584200001, + "workflow_seconds": null, + "total_seconds": 39.64186499400002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001766992999989725, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 39.64186499400002 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174442", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20117491299998846, + "build_seconds": 38.363812155000005, + "workflow_seconds": null, + "total_seconds": 38.56498706799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6816028860000074, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.56498706799999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174441", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/report.md b/results/msgpack-sbt/cadence-rolling-seed/report.md new file mode 100644 index 000000000..9f1865671 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# msgpack-sbt: cadence-rolling-seed + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10111915200000965 | 39.54074584200001 | 39.64186499400002 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20117491299998846 | 38.363812155000005 | 38.56498706799999 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6c6e7fa49 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,147 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111915200000965, + "build_seconds": 39.54074584200001, + "workflow_seconds": null, + "total_seconds": 39.64186499400002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001766992999989725, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 39.64186499400002 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174442", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..661540de1 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,149 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "upstream-head", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20117491299998846, + "build_seconds": 38.363812155000005, + "workflow_seconds": null, + "total_seconds": 38.56498706799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6816028860000074, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 38.56498706799999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-rolling-cadence-20261005-msgpack-sbt", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322609279", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174441", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322609279-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322609279" + ] +} diff --git a/results/msgpack-sbt/cadence-rolling-seed/series.json b/results/msgpack-sbt/cadence-rolling-seed/series.json new file mode 100644 index 000000000..0afee4648 --- /dev/null +++ b/results/msgpack-sbt/cadence-rolling-seed/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "341246f8f54281a887bc161ede26db9479390372429d49012c2db909a153b614", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-msgpack-sbt" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "e563c124fae3f2f6a44f5c7921ea7dfc79ded103d502dd402e643463bab6cde7" +} diff --git a/results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json b/results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json new file mode 100644 index 000000000..e5d20102c --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/completions/37323140444.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-distroless", + "sample": 1, + "run_id": "37323140444", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e400914e058065b703dd4877a9bc49d2827c428e0a1c6b68c4f0a512fa8b3487" +} diff --git a/results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..ae0e4e952 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless", + "series_id": "cadence-rolling-advance-distroless", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:43Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323140444, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444" +} diff --git a/results/n8n/cadence-rolling-advance-distroless/report.json b/results/n8n/cadence-rolling-advance-distroless/report.json new file mode 100644 index 000000000..7ea6f0ee4 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "81a8042363d108962966347f75d613c85f5c624d58fcbec68a25b3563c1a2082", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-distroless", + "sample": 1, + "run_id": "37323140444", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323140444", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e400914e058065b703dd4877a9bc49d2827c428e0a1c6b68c4f0a512fa8b3487" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 128, + "min": 128, + "max": 128 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 87, + "min": 87, + "max": 87 + }, + "storage": { + "median": 581233979, + "min": 581233979, + "max": 581233979 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 119, + "workflow_seconds": 285, + "total_seconds": 128, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 128 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174476", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 87, + "workflow_seconds": 249, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581233979, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581233979, + "unmeasured_tags": [], + "total_bytes": 581233979, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "1c175ac8-a90f-48fc-8ce3-65c683d162ed", + "manifest_root_digest": "sha256:24200a6466d459f8679011ec2e110e39bef6ef8901722551303446c3b13fda40", + "kv_total_size": 0, + "compressed_size": 581233979, + "size": 581233979 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174477", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-distroless/report.md b/results/n8n/cadence-rolling-advance-distroless/report.md new file mode 100644 index 000000000..66c68035c --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-distroless + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 128 | 128–128 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 87 | 87–87 | 581233979 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 9 | 119 | 128 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 87 | 87 | 581233979 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..e44eecc41 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 119, + "workflow_seconds": 285, + "total_seconds": 128, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 128 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174476", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] +} diff --git a/results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6125cade9 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 87, + "workflow_seconds": 249, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581233979, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581233979, + "unmeasured_tags": [], + "total_bytes": 581233979, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "1c175ac8-a90f-48fc-8ce3-65c683d162ed", + "manifest_root_digest": "sha256:24200a6466d459f8679011ec2e110e39bef6ef8901722551303446c3b13fda40", + "kv_total_size": 0, + "compressed_size": 581233979, + "size": 581233979 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323140444", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174477", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323140444-1", + "series": { + "id": "cadence-rolling-advance-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323140444" + ] +} diff --git a/results/n8n/cadence-rolling-advance-distroless/series.json b/results/n8n/cadence-rolling-advance-distroless/series.json new file mode 100644 index 000000000..9114f45fb --- /dev/null +++ b/results/n8n/cadence-rolling-advance-distroless/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-distroless", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "distroless", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "81a8042363d108962966347f75d613c85f5c624d58fcbec68a25b3563c1a2082" +} diff --git a/results/n8n/cadence-rolling-advance-docker/completions/37323145174.json b/results/n8n/cadence-rolling-advance-docker/completions/37323145174.json new file mode 100644 index 000000000..8ffe329a1 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/completions/37323145174.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-docker", + "sample": 1, + "run_id": "37323145174", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ad802f0c8845d5f11e782410c92ba8ef0ac2ac46866b617108445404349e986" +} diff --git a/results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..50ad29f00 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker", + "series_id": "cadence-rolling-advance-docker", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:45Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323145174, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174" +} diff --git a/results/n8n/cadence-rolling-advance-docker/report.json b/results/n8n/cadence-rolling-advance-docker/report.json new file mode 100644 index 000000000..078997ef7 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "85ee37fee93a651e47f12c7a5fff50734381e919b02763f2c371359baa239de3", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-docker", + "sample": 1, + "run_id": "37323145174", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323145174", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6ad802f0c8845d5f11e782410c92ba8ef0ac2ac46866b617108445404349e986" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 178, + "min": 178, + "max": 178 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 152, + "min": 152, + "max": 152 + }, + "storage": { + "median": 483880710, + "min": 483880710, + "max": 483880710 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 173, + "workflow_seconds": 384, + "total_seconds": 178, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 178 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174488", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 152, + "workflow_seconds": 351, + "total_seconds": 152, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 152 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483880710, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483880710, + "unmeasured_tags": [], + "total_bytes": 483880710, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "68ec16be-5a23-408a-aaef-3d4cdcd6d219", + "manifest_root_digest": "sha256:340b591e8568dcd06731dc4a0585f64e5c735a7da2b952f6eb31614e1190c243", + "kv_total_size": 0, + "compressed_size": 483880710, + "size": 483880710 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174489", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-docker/report.md b/results/n8n/cadence-rolling-advance-docker/report.md new file mode 100644 index 000000000..be60ebd00 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-docker + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 178 | 178–178 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 152 | 152–152 | 483880710 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 173 | 178 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 152 | 152 | 483880710 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..89257230f --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 173, + "workflow_seconds": 384, + "total_seconds": 178, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 178 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174488", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] +} diff --git a/results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6b4ff6239 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 152, + "workflow_seconds": 351, + "total_seconds": 152, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 152 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483880710, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483880710, + "unmeasured_tags": [], + "total_bytes": 483880710, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "68ec16be-5a23-408a-aaef-3d4cdcd6d219", + "manifest_root_digest": "sha256:340b591e8568dcd06731dc4a0585f64e5c735a7da2b952f6eb31614e1190c243", + "kv_total_size": 0, + "compressed_size": 483880710, + "size": 483880710 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323145174", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174489", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323145174-1", + "series": { + "id": "cadence-rolling-advance-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323145174" + ] +} diff --git a/results/n8n/cadence-rolling-advance-docker/series.json b/results/n8n/cadence-rolling-advance-docker/series.json new file mode 100644 index 000000000..557e3ae0f --- /dev/null +++ b/results/n8n/cadence-rolling-advance-docker/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-docker", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "docker", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "85ee37fee93a651e47f12c7a5fff50734381e919b02763f2c371359baa239de3" +} diff --git a/results/n8n/cadence-rolling-advance-runners/completions/37323150431.json b/results/n8n/cadence-rolling-advance-runners/completions/37323150431.json new file mode 100644 index 000000000..758123266 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/completions/37323150431.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-runners", + "sample": 1, + "run_id": "37323150431", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "9b87465fe9de14a641423ad22393b66514278972c78f5aab173864f341c67a5f" +} diff --git a/results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..bcd69f9b8 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners", + "series_id": "cadence-rolling-advance-runners", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:47Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323150431, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431" +} diff --git a/results/n8n/cadence-rolling-advance-runners/report.json b/results/n8n/cadence-rolling-advance-runners/report.json new file mode 100644 index 000000000..130d20deb --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "e7626673b7f2595136f07f3f3953bffe5f2e1d64ecf8b9775519da47bf0aa605", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-runners", + "sample": 1, + "run_id": "37323150431", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323150431", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "9b87465fe9de14a641423ad22393b66514278972c78f5aab173864f341c67a5f" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 108, + "min": 108, + "max": 108 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 59, + "min": 59, + "max": 59 + }, + "storage": { + "median": 413978377, + "min": 413978377, + "max": 413978377 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 98, + "workflow_seconds": 308, + "total_seconds": 108, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 108 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174478", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 59, + "workflow_seconds": 256, + "total_seconds": 59, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 59 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413978377, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413978377, + "unmeasured_tags": [], + "total_bytes": 413978377, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9e8e1ca5-b03b-4a29-9314-be454f6d3877", + "manifest_root_digest": "sha256:529ae2cf71c27d48484f8415c6e120b65d63caabbf808d1d1b48e18dd006c741", + "kv_total_size": 0, + "compressed_size": 413978377, + "size": 413978377 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174479", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-runners/report.md b/results/n8n/cadence-rolling-advance-runners/report.md new file mode 100644 index 000000000..c25560fb1 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-runners + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 108 | 108–108 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 59 | 59–59 | 413978377 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 10 | 98 | 108 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 59 | 59 | 413978377 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..ca7dbf9ed --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 98, + "workflow_seconds": 308, + "total_seconds": 108, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 108 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174478", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] +} diff --git a/results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json new file mode 100644 index 000000000..c0b350957 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 59, + "workflow_seconds": 256, + "total_seconds": 59, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 59 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413978377, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413978377, + "unmeasured_tags": [], + "total_bytes": 413978377, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9e8e1ca5-b03b-4a29-9314-be454f6d3877", + "manifest_root_digest": "sha256:529ae2cf71c27d48484f8415c6e120b65d63caabbf808d1d1b48e18dd006c741", + "kv_total_size": 0, + "compressed_size": 413978377, + "size": 413978377 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323150431", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174479", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323150431-1", + "series": { + "id": "cadence-rolling-advance-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323150431" + ] +} diff --git a/results/n8n/cadence-rolling-advance-runners/series.json b/results/n8n/cadence-rolling-advance-runners/series.json new file mode 100644 index 000000000..d247d0e8a --- /dev/null +++ b/results/n8n/cadence-rolling-advance-runners/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-runners", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "runners", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "e7626673b7f2595136f07f3f3953bffe5f2e1d64ecf8b9775519da47bf0aa605" +} diff --git a/results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json b/results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json new file mode 100644 index 000000000..f12a3eaef --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/completions/37323155106.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-turbo", + "sample": 1, + "run_id": "37323155106", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e0dcca70dfbc76d3d4ea52feccbdaf0c6a47ad2111e0432ca3c184010a6920f9" +} diff --git a/results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..5e3702c7b --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo", + "series_id": "cadence-rolling-advance-turbo", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:49Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323155106, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106" +} diff --git a/results/n8n/cadence-rolling-advance-turbo/report.json b/results/n8n/cadence-rolling-advance-turbo/report.json new file mode 100644 index 000000000..7be9956eb --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/report.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "plan_sha256": "b4b90237fb539c9315406c5e54501cb951d86de26a04509e10b3086b20bc4e24", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-turbo", + "sample": 1, + "run_id": "37323155106", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323155106", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e0dcca70dfbc76d3d4ea52feccbdaf0c6a47ad2111e0432ca3c184010a6920f9" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 134, + "min": 134, + "max": 134 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 131, + "min": 131, + "max": 131 + }, + "storage": { + "median": 96876410, + "min": 96876410, + "max": 96876410 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 133, + "workflow_seconds": 170, + "total_seconds": 134, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 134 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-206fc6f317e353a98f7359801376a22d6bd136a2", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174482", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 124, + "workflow_seconds": 166, + "total_seconds": 131, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 131 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 96876410, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 96876410, + "unmeasured_tags": [], + "total_bytes": 96876410, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 96876410, + "compressed_size": 96876410, + "size": 96876410 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174483", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-advance-turbo/report.md b/results/n8n/cadence-rolling-advance-turbo/report.md new file mode 100644 index 000000000..fc032cdf5 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-advance-turbo + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 134 | 134–134 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 131 | 131–131 | 96876410 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 1 | 133 | 134 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 7 | 124 | 131 | 96876410 | boringcache-check | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..2ade4d40f --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 133, + "workflow_seconds": 170, + "total_seconds": 134, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 134 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-206fc6f317e353a98f7359801376a22d6bd136a2", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174482", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] +} diff --git a/results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6c38b5815 --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/runs/1-boringcache-commit.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 124, + "workflow_seconds": 166, + "total_seconds": 131, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 131 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 96876410, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 96876410, + "unmeasured_tags": [], + "total_bytes": 96876410, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 96876410, + "compressed_size": 96876410, + "size": 96876410 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "206fc6f317e353a98f7359801376a22d6bd136a2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323155106", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174483", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323155106-1", + "series": { + "id": "cadence-rolling-advance-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323155106" + ] +} diff --git a/results/n8n/cadence-rolling-advance-turbo/series.json b/results/n8n/cadence-rolling-advance-turbo/series.json new file mode 100644 index 000000000..1859839ab --- /dev/null +++ b/results/n8n/cadence-rolling-advance-turbo/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-advance-turbo", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "39c7440d0ecceca994df1dbeb3c7953d4571dd8fef30105f1639c50e53b17de5", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "206fc6f317e353a98f7359801376a22d6bd136a2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "b4b90237fb539c9315406c5e54501cb951d86de26a04509e10b3086b20bc4e24" +} diff --git a/results/n8n/cadence-rolling-next-turbo/completions/37325183094.json b/results/n8n/cadence-rolling-next-turbo/completions/37325183094.json new file mode 100644 index 000000000..f6b0597e7 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/completions/37325183094.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-next-turbo", + "sample": 1, + "run_id": "37325183094", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094", + "source_sha": "173fbed3116e31cad526e2e4d777f3f8594c47ce", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1a7d6899fa0e314c6a68699f3312ef3c9e51b950b3a62522960b1e371057e50" +} diff --git a/results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0a8ba9426 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo", + "series_id": "cadence-rolling-next-turbo", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:29:56Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325183094, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094" +} diff --git a/results/n8n/cadence-rolling-next-turbo/report.json b/results/n8n/cadence-rolling-next-turbo/report.json new file mode 100644 index 000000000..33c3d89a9 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/report.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "plan_sha256": "b749a682eb42d6c7a423da19c572b9ccd92b020e2c0b11b072e4da3d6a1884c1", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-next-turbo", + "sample": 1, + "run_id": "37325183094", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325183094", + "source_sha": "173fbed3116e31cad526e2e4d777f3f8594c47ce", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1a7d6899fa0e314c6a68699f3312ef3c9e51b950b3a62522960b1e371057e50" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 23, + "min": 23, + "max": 23 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": { + "median": 102947794, + "min": 102947794, + "max": 102947794 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 21, + "workflow_seconds": 60, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174594", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 23, + "workflow_seconds": 67, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 102947794, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 102947794, + "unmeasured_tags": [], + "total_bytes": 102947794, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 102947794, + "compressed_size": 102947794, + "size": 102947794 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174595", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-next-turbo/report.md b/results/n8n/cadence-rolling-next-turbo/report.md new file mode 100644 index 000000000..cf464c1d3 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-next-turbo + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 23 | 23–23 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 28 | 28–28 | 102947794 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 2 | 21 | 23 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 5 | 23 | 28 | 102947794 | boringcache-check | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..0386ced48 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 21, + "workflow_seconds": 60, + "total_seconds": 23, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 23 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174594", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] +} diff --git a/results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json new file mode 100644 index 000000000..a2dc5c05d --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/runs/1-boringcache-commit.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 23, + "workflow_seconds": 67, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 102947794, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 102947794, + "unmeasured_tags": [], + "total_bytes": 102947794, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 102947794, + "compressed_size": 102947794, + "size": 102947794 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325183094", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174595", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325183094-1", + "series": { + "id": "cadence-rolling-next-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325183094" + ] +} diff --git a/results/n8n/cadence-rolling-next-turbo/series.json b/results/n8n/cadence-rolling-next-turbo/series.json new file mode 100644 index 000000000..ff065fef0 --- /dev/null +++ b/results/n8n/cadence-rolling-next-turbo/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-next-turbo", + "created_at": "2026-10-05T14:29:53Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "aa1b250356703b39cff4a2a43121821f438f82220e94fc93d4d0a976f79e01f9", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "b7071c91c7ef959e1cc3d703495e4fee9fcc9744", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "b749a682eb42d6c7a423da19c572b9ccd92b020e2c0b11b072e4da3d6a1884c1" +} diff --git a/results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json b/results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json new file mode 100644 index 000000000..84b47892e --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/completions/37321753318.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-distroless", + "sample": 1, + "run_id": "37321753318", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5dba4af82ea3265cfe8253b205a2ab4ede49bb19b33981edd5405fe8cd44d094" +} diff --git a/results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..ce1189de7 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless", + "series_id": "cadence-rolling-seed-distroless", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:22Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321753318, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318" +} diff --git a/results/n8n/cadence-rolling-seed-distroless/report.json b/results/n8n/cadence-rolling-seed-distroless/report.json new file mode 100644 index 000000000..45a155dfc --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "d1f18342eadd50145421ea9ed42dd7383e8f463fae9a7416ef1d5661c4c3c73b", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-distroless", + "sample": 1, + "run_id": "37321753318", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321753318", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5dba4af82ea3265cfe8253b205a2ab4ede49bb19b33981edd5405fe8cd44d094" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 142, + "min": 142, + "max": 142 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 85, + "min": 85, + "max": 85 + }, + "storage": { + "median": 581058531, + "min": 581058531, + "max": 581058531 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 136, + "workflow_seconds": 344, + "total_seconds": 142, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 142 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174357", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 283, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581058531, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581058531, + "unmeasured_tags": [], + "total_bytes": 581058531, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b2e72ac8-7460-474a-bb9e-ed933014c045", + "manifest_root_digest": "sha256:6969aef351b47984ed9c2d79840309b8afbe28b4fcad348e3292ed70fec47403", + "kv_total_size": 0, + "compressed_size": 581058531, + "size": 581058531 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174358", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-distroless/report.md b/results/n8n/cadence-rolling-seed-distroless/report.md new file mode 100644 index 000000000..dc463bf77 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-distroless + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 142 | 142–142 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 85 | 85–85 | 581058531 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 6 | 136 | 142 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 85 | 85 | 581058531 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..ebd380c68 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 136, + "workflow_seconds": 344, + "total_seconds": 142, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 142 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174357", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] +} diff --git a/results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json new file mode 100644 index 000000000..85b486eb9 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 283, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581058531, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581058531, + "unmeasured_tags": [], + "total_bytes": 581058531, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "tag": "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b2e72ac8-7460-474a-bb9e-ed933014c045", + "manifest_root_digest": "sha256:6969aef351b47984ed9c2d79840309b8afbe28b4fcad348e3292ed70fec47403", + "kv_total_size": 0, + "compressed_size": 581058531, + "size": 581058531 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-rolling-cadence-20261005-n8n-distroless-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321753318", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174358", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321753318-1", + "series": { + "id": "cadence-rolling-seed-distroless", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321753318" + ] +} diff --git a/results/n8n/cadence-rolling-seed-distroless/series.json b/results/n8n/cadence-rolling-seed-distroless/series.json new file mode 100644 index 000000000..975d204ec --- /dev/null +++ b/results/n8n/cadence-rolling-seed-distroless/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-distroless", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "distroless", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-distroless", + "variant": "distroless" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "d1f18342eadd50145421ea9ed42dd7383e8f463fae9a7416ef1d5661c4c3c73b" +} diff --git a/results/n8n/cadence-rolling-seed-docker/completions/37321758035.json b/results/n8n/cadence-rolling-seed-docker/completions/37321758035.json new file mode 100644 index 000000000..0e48a1c95 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/completions/37321758035.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-docker", + "sample": 1, + "run_id": "37321758035", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a282b7867dd2a2cd6c278574761db7beba802f44b877dbacbba2a0058a8010bf" +} diff --git a/results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0a3662357 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker", + "series_id": "cadence-rolling-seed-docker", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:24Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321758035, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035" +} diff --git a/results/n8n/cadence-rolling-seed-docker/report.json b/results/n8n/cadence-rolling-seed-docker/report.json new file mode 100644 index 000000000..0a92caec3 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "5cf26022a9d254a005a5fa1b728266bc712b1b1735d8358d13f3b62629404d3f", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-docker", + "sample": 1, + "run_id": "37321758035", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321758035", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a282b7867dd2a2cd6c278574761db7beba802f44b877dbacbba2a0058a8010bf" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 361, + "min": 361, + "max": 361 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 233, + "min": 233, + "max": 233 + }, + "storage": { + "median": 483504441, + "min": 483504441, + "max": 483504441 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 352, + "workflow_seconds": 553, + "total_seconds": 361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 361 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174360", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 232, + "workflow_seconds": 399, + "total_seconds": 233, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 233 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483504441, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483504441, + "unmeasured_tags": [], + "total_bytes": 483504441, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "52702ba9-0b6b-4974-be8a-8ac5b7dd91dd", + "manifest_root_digest": "sha256:bc0cec58d12408d3a0ea3da54ac0dfb309704a63e3da68c8c89acf792299c800", + "kv_total_size": 0, + "compressed_size": 483504441, + "size": 483504441 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174359", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-docker/report.md b/results/n8n/cadence-rolling-seed-docker/report.md new file mode 100644 index 000000000..bb3a8ba40 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-docker + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 361 | 361–361 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 233 | 233–233 | 483504441 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 9 | 352 | 361 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 1 | 232 | 233 | 483504441 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..1da972959 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 352, + "workflow_seconds": 553, + "total_seconds": 361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 361 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-rolling-cadence-20261005-n8n-docker", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174360", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] +} diff --git a/results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json new file mode 100644 index 000000000..6c0b63bda --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 232, + "workflow_seconds": 399, + "total_seconds": 233, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 233 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + }, + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483504441, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483504441, + "unmeasured_tags": [], + "total_bytes": 483504441, + "observations": [ + { + "requested_tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "tag": "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "52702ba9-0b6b-4974-be8a-8ac5b7dd91dd", + "manifest_root_digest": "sha256:bc0cec58d12408d3a0ea3da54ac0dfb309704a63e3da68c8c89acf792299c800", + "kv_total_size": 0, + "compressed_size": 483504441, + "size": 483504441 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-rolling-cadence-20261005-n8n-docker-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321758035", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174359", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321758035-1", + "series": { + "id": "cadence-rolling-seed-docker", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321758035" + ] +} diff --git a/results/n8n/cadence-rolling-seed-docker/series.json b/results/n8n/cadence-rolling-seed-docker/series.json new file mode 100644 index 000000000..df8231c07 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-docker/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-docker", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "docker", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-docker", + "variant": "docker" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "5cf26022a9d254a005a5fa1b728266bc712b1b1735d8358d13f3b62629404d3f" +} diff --git a/results/n8n/cadence-rolling-seed-runners/completions/37321762921.json b/results/n8n/cadence-rolling-seed-runners/completions/37321762921.json new file mode 100644 index 000000000..5728911b1 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/completions/37321762921.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-runners", + "sample": 1, + "run_id": "37321762921", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8f8db69bea3b4b1a7cc45f393989c159db64f5e95d90152f7beb9484b6382961" +} diff --git a/results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..c0c90d6e4 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners", + "series_id": "cadence-rolling-seed-runners", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:26Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321762921, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921" +} diff --git a/results/n8n/cadence-rolling-seed-runners/report.json b/results/n8n/cadence-rolling-seed-runners/report.json new file mode 100644 index 000000000..0ceaff2a9 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/report.json @@ -0,0 +1,573 @@ +{ + "schema_version": 1, + "plan_sha256": "1755d2d8b22261156f9b44be5b049be4d4517dfdede31be0c5662ab37afe6b4f", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-runners", + "sample": 1, + "run_id": "37321762921", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321762921", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8f8db69bea3b4b1a7cc45f393989c159db64f5e95d90152f7beb9484b6382961" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 97, + "min": 97, + "max": 97 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 60, + "min": 60, + "max": 60 + }, + "storage": { + "median": 413880460, + "min": 413880460, + "max": 413880460 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 92, + "workflow_seconds": 297, + "total_seconds": 97, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 97 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174362", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 60, + "workflow_seconds": 217, + "total_seconds": 60, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 60 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413880460, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413880460, + "unmeasured_tags": [], + "total_bytes": 413880460, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eaa46e3-222f-44aa-9cae-ea0496dc904c", + "manifest_root_digest": "sha256:8e9622acb5f3e6a361fc412300476b4c657ff9351ef1e3a458b78995770c188f", + "kv_total_size": 0, + "compressed_size": 413880460, + "size": 413880460 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174361", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-runners/report.md b/results/n8n/cadence-rolling-seed-runners/report.md new file mode 100644 index 000000000..df8fd6d39 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-runners + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 97 | 97–97 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 60 | 60–60 | 413880460 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 92 | 97 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 60 | 60 | 413880460 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..87163af3c --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 92, + "workflow_seconds": 297, + "total_seconds": 97, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 97 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-rolling-cadence-20261005-n8n-runners", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174362", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] +} diff --git a/results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json new file mode 100644 index 000000000..45a4c65eb --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/runs/1-boringcache-commit.json @@ -0,0 +1,277 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 60, + "workflow_seconds": 217, + "total_seconds": 60, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 60 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + }, + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413880460, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413880460, + "unmeasured_tags": [], + "total_bytes": 413880460, + "observations": [ + { + "requested_tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "tag": "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9eaa46e3-222f-44aa-9cae-ea0496dc904c", + "manifest_root_digest": "sha256:8e9622acb5f3e6a361fc412300476b4c657ff9351ef1e3a458b78995770c188f", + "kv_total_size": 0, + "compressed_size": 413880460, + "size": 413880460 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-rolling-cadence-20261005-n8n-runners-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321762921", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174361", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321762921-1", + "series": { + "id": "cadence-rolling-seed-runners", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321762921" + ] +} diff --git a/results/n8n/cadence-rolling-seed-runners/series.json b/results/n8n/cadence-rolling-seed-runners/series.json new file mode 100644 index 000000000..966a8211d --- /dev/null +++ b/results/n8n/cadence-rolling-seed-runners/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-runners", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "runners", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-runners", + "variant": "runners" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "1755d2d8b22261156f9b44be5b049be4d4517dfdede31be0c5662ab37afe6b4f" +} diff --git a/results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json b/results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json new file mode 100644 index 000000000..c8bb26ea0 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/completions/37321767409.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-turbo", + "sample": 1, + "run_id": "37321767409", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c5788a3d03b981f520010fe3cc13c99e5d16a39606f0ff7b0d9437bbb316174c" +} diff --git a/results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json b/results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..29484dbf8 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo", + "series_id": "cadence-rolling-seed-turbo", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:28Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321767409, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409" +} diff --git a/results/n8n/cadence-rolling-seed-turbo/report.json b/results/n8n/cadence-rolling-seed-turbo/report.json new file mode 100644 index 000000000..7e9a1c4c7 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/report.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "plan_sha256": "844a1fe6776711f983b947113353f8af9d82dc08f2f07180291060668cdfc0fb", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-turbo", + "sample": 1, + "run_id": "37321767409", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321767409", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c5788a3d03b981f520010fe3cc13c99e5d16a39606f0ff7b0d9437bbb316174c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 85, + "min": 85, + "max": 85 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 87, + "min": 87, + "max": 87 + }, + "storage": { + "median": 46411206, + "min": 46411206, + "max": 46411206 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 125, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174364", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 3, + "build_seconds": 84, + "workflow_seconds": 127, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46411206, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 46411206, + "unmeasured_tags": [], + "total_bytes": 46411206, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46411206, + "compressed_size": 46411206, + "size": 46411206 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174365", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/cadence-rolling-seed-turbo/report.md b/results/n8n/cadence-rolling-seed-turbo/report.md new file mode 100644 index 000000000..8cb51cff7 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/report.md @@ -0,0 +1,29 @@ +# n8n: cadence-rolling-seed-turbo + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 85 | 85–85 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 87 | 87–87 | 46411206 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 85 | 85 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 3 | 84 | 87 | 46411206 | boringcache-check | miss | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json b/results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..5262645fe --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/runs/1-actions-cache-commit.json @@ -0,0 +1,231 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 85, + "workflow_seconds": 125, + "total_seconds": 85, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 85 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-rolling-cadence-20261005-n8n-turbo-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174364", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] +} diff --git a/results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json b/results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json new file mode 100644 index 000000000..4b246ba3f --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/runs/1-boringcache-commit.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 3, + "build_seconds": 84, + "workflow_seconds": 127, + "total_seconds": 87, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 87 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46411206, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 46411206, + "unmeasured_tags": [], + "total_bytes": 46411206, + "observations": [ + { + "requested_tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "tag": "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46411206, + "compressed_size": 46411206, + "size": 46411206 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-rolling-cadence-20261005-n8n-turbo-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321767409", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174365", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321767409-1", + "series": { + "id": "cadence-rolling-seed-turbo", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321767409" + ] +} diff --git a/results/n8n/cadence-rolling-seed-turbo/series.json b/results/n8n/cadence-rolling-seed-turbo/series.json new file mode 100644 index 000000000..545a32472 --- /dev/null +++ b/results/n8n/cadence-rolling-seed-turbo/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "cadence-rolling-seed-turbo", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "855f813acceea9d226331424790fefd7ca50432b273f5427087fef51cde03465", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-n8n-turbo", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "844a1fe6776711f983b947113353f8af9d82dc08f2f07180291060668cdfc0fb" +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json b/results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json new file mode 100644 index 000000000..1872773b6 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/completions/37324219008.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-ccache-02", + "sample": 1, + "run_id": "37324219008", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c007e7636ce1312c0b26401c97fbfcf189649f4579cdb231c7476c47bf7612cc" +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..4830c9f91 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache", + "series_id": "cadence-rolling-advance-ccache-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:22:45Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324219008, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008" +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/report.json b/results/obs-studio/cadence-rolling-advance-ccache-02/report.json new file mode 100644 index 000000000..9e1babbb9 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/report.json @@ -0,0 +1,490 @@ +{ + "schema_version": 1, + "plan_sha256": "8587641676089f6569ab4c5ef0cd2bd84894ac5368e74ac41018d41a1be9541b", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-ccache-02", + "sample": 1, + "run_id": "37324219008", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324219008", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c007e7636ce1312c0b26401c97fbfcf189649f4579cdb231c7476c47bf7612cc" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 39.52239645200001, + "min": 39.52239645200001, + "max": 39.52239645200001 + }, + "storage": { + "median": 164323972, + "min": 164323972, + "max": 164323972 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 57.526410286000015, + "min": 57.526410286000015, + "max": 57.526410286000015 + }, + "storage": { + "median": 176275703, + "min": 176275703, + "max": 176275703 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 37.52239645200001, + "total_seconds": 39.52239645200001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 39.52239645200001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 164323972, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "complete": true, + "measured_bytes": 164323972, + "total_bytes": 164323972, + "observations": [ + { + "id": 8523145597, + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "a6730e8f7256fcb7bacf34155bb5305481c02b0e3925c471e4929ecf01b84d07", + "created_at": "2026-10-05T14:30:17.029024Z", + "last_accessed_at": "2026-10-05T14:31:09.615918Z", + "size_in_bytes": 164323972 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174597", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 49.526410286000015, + "total_seconds": 57.526410286000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 57.526410286000015 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 176275703, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 176275703, + "unmeasured_tags": [], + "total_bytes": 176275703, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 176275703, + "compressed_size": 176275703, + "size": 176275703 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174598", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/report.md b/results/obs-studio/cadence-rolling-advance-ccache-02/report.md new file mode 100644 index 000000000..74db081f8 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-advance-ccache-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 39.52239645200001 | 39.52239645200001–39.52239645200001 | 164323972 | 1 | +| Changed-source build | BoringCache | 1 | 57.526410286000015 | 57.526410286000015–57.526410286000015 | 176275703 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 2 | 37.52239645200001 | 39.52239645200001 | 164323972 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 8 | 49.526410286000015 | 57.526410286000015 | 176275703 | boringcache-check | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..70335a6c2 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-actions-cache-commit.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 37.52239645200001, + "total_seconds": 39.52239645200001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 39.52239645200001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 164323972, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "complete": true, + "measured_bytes": 164323972, + "total_bytes": 164323972, + "observations": [ + { + "id": 8523145597, + "key": "gha-obs-ccache-cadence-20261005-obs-studio-ccache-v2-Linux-X64-37323465858-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "a6730e8f7256fcb7bacf34155bb5305481c02b0e3925c471e4929ecf01b84d07", + "created_at": "2026-10-05T14:30:17.029024Z", + "last_accessed_at": "2026-10-05T14:31:09.615918Z", + "size_in_bytes": 164323972 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174597", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..86f069ffe --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 49.526410286000015, + "total_seconds": 57.526410286000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 57.526410286000015 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 176275703, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 176275703, + "unmeasured_tags": [], + "total_bytes": 176275703, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 176275703, + "compressed_size": 176275703, + "size": 176275703 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324219008", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174598", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37324219008-1", + "series": { + "id": "cadence-rolling-advance-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324219008" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-ccache-02/series.json b/results/obs-studio/cadence-rolling-advance-ccache-02/series.json new file mode 100644 index 000000000..e08b3b383 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-ccache-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-ccache-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e07eb36f9b74e7fb6d33f5fb4c58de6663a6ec8c8319c4ae3df273cdf2079d33", + "variant": "ccache", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "8587641676089f6569ab4c5ef0cd2bd84894ac5368e74ac41018d41a1be9541b" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json b/results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json new file mode 100644 index 000000000..b4f677c8d --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/completions/37326340221.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode-02", + "sample": 1, + "run_id": "37326340221", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221", + "source_sha": "eaa2b54bf4d09870c8f139850bd2930a28b51f84", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3697df862b900d7681b5db50c705e57271393a0d0b8a050ec75b69e7644f9754" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..a64899f71 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode", + "series_id": "cadence-rolling-advance-xcode-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:38:30Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37326340221, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/report.json b/results/obs-studio/cadence-rolling-advance-xcode-02/report.json new file mode 100644 index 000000000..ad2db42d5 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/report.json @@ -0,0 +1,490 @@ +{ + "schema_version": 1, + "plan_sha256": "71a4a39bcc095f6bcf55fbb038323631597c2974ccb8b14c437ed5ee04d4c9d8", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode-02", + "sample": 1, + "run_id": "37326340221", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326340221", + "source_sha": "eaa2b54bf4d09870c8f139850bd2930a28b51f84", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3697df862b900d7681b5db50c705e57271393a0d0b8a050ec75b69e7644f9754" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 189.99793799999998, + "min": 189.99793799999998, + "max": 189.99793799999998 + }, + "storage": { + "median": 309445116, + "min": 309445116, + "max": 309445116 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 169.57710600000001, + "min": 169.57710600000001, + "max": 169.57710600000001 + }, + "storage": { + "median": 1004724519, + "min": 1004724519, + "max": 1004724519 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 178.99793799999998, + "total_seconds": 189.99793799999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 189.99793799999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 309445116, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "complete": true, + "measured_bytes": 309445116, + "total_bytes": 309445116, + "observations": [ + { + "id": 8524219730, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:51:56.940267Z", + "last_accessed_at": "2026-10-05T14:53:14.694488Z", + "size_in_bytes": 309445116 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174657", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 23, + "build_seconds": 146.57710600000001, + "total_seconds": 169.57710600000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 169.57710600000001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1004724519, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 1004724519, + "unmeasured_tags": [], + "total_bytes": 1004724519, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 1004724519, + "compressed_size": 1004724519, + "size": 1004724519 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174656", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/report.md b/results/obs-studio/cadence-rolling-advance-xcode-02/report.md new file mode 100644 index 000000000..82ae5f62f --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-advance-xcode-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 189.99793799999998 | 189.99793799999998–189.99793799999998 | 309445116 | 1 | +| Changed-source build | BoringCache | 1 | 169.57710600000001 | 169.57710600000001–169.57710600000001 | 1004724519 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 11 | 178.99793799999998 | 189.99793799999998 | 309445116 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 23 | 146.57710600000001 | 169.57710600000001 | 1004724519 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..53e9d8a4f --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-actions-cache-commit.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 178.99793799999998, + "total_seconds": 189.99793799999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 189.99793799999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 309445116, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "complete": true, + "measured_bytes": 309445116, + "total_bytes": 309445116, + "observations": [ + { + "id": 8524219730, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-v2-macOS-ARM64-37326133177-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:51:56.940267Z", + "last_accessed_at": "2026-10-05T14:53:14.694488Z", + "size_in_bytes": 309445116 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174657", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f846478da --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 23, + "build_seconds": 146.57710600000001, + "total_seconds": 169.57710600000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 169.57710600000001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1004724519, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 1004724519, + "unmeasured_tags": [], + "total_bytes": 1004724519, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 1004724519, + "compressed_size": 1004724519, + "size": 1004724519 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326340221", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174656", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326340221-1", + "series": { + "id": "cadence-rolling-advance-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326340221" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode-02/series.json b/results/obs-studio/cadence-rolling-advance-xcode-02/series.json new file mode 100644 index 000000000..b9c56b27a --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode-02", + "created_at": "2026-10-05T14:38:27Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "8f533cf9f917db17ddc72b7108b9666ddecaf44002a0ab817ee63a9e55c4348d", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "71a4a39bcc095f6bcf55fbb038323631597c2974ccb8b14c437ed5ee04d4c9d8" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json b/results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json new file mode 100644 index 000000000..280adaeca --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/completions/37323234591.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "sample": 1, + "run_id": "37323234591", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "affe5bec73070abd855bf7db61b49d6eeb974d240029ffa55c02074c2486b458" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..c2a379091 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode", + "series_id": "cadence-rolling-advance-xcode", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:15:23Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323234591, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591" +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/report.json b/results/obs-studio/cadence-rolling-advance-xcode/report.json new file mode 100644 index 000000000..0d4426c03 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/report.json @@ -0,0 +1,278 @@ +{ + "schema_version": 1, + "plan_sha256": "c42ea925dc2d15cb5a24ef14e27b267001b6bf8b9ed29989e51c7e927e3875e4", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "sample": 1, + "run_id": "37323234591", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "affe5bec73070abd855bf7db61b49d6eeb974d240029ffa55c02074c2486b458" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "sample": 1, + "run_id": "37323234591", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323234591", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "affe5bec73070abd855bf7db61b49d6eeb974d240029ffa55c02074c2486b458" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 129.465303, + "min": 129.465303, + "max": 129.465303 + }, + "storage": { + "median": 310246072, + "min": 310246072, + "max": 310246072 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "215aea66ef6c9adae46be8c8f6d89395e0ffdffcb64b7184ef9f0157512506cc", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 124.465303, + "total_seconds": 129.465303, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 129.465303 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 310246072, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "complete": true, + "measured_bytes": 310246072, + "total_bytes": 310246072, + "observations": [ + { + "id": 8522777408, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:22:57.114691Z", + "last_accessed_at": "2026-10-05T14:26:07.297970Z", + "size_in_bytes": 310246072 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323234591", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174567", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37323234591-1", + "series": { + "id": "cadence-rolling-advance-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323234591" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/report.md b/results/obs-studio/cadence-rolling-advance-xcode/report.md new file mode 100644 index 000000000..272a5f564 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/report.md @@ -0,0 +1,37 @@ +# obs-studio: cadence-rolling-advance-xcode + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 129.465303 | 129.465303–129.465303 | 310246072 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +Run 37323234591 failed completion checks: + +- Workflow concluded failure +- BoringCache obs-studio xcode commit: failure +- obs-studio rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 5 | 124.465303 | 129.465303 | 310246072 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..1e06e516e --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/runs/1-actions-cache-commit.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "215aea66ef6c9adae46be8c8f6d89395e0ffdffcb64b7184ef9f0157512506cc", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 124.465303, + "total_seconds": 129.465303, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 129.465303 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 310246072, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "complete": true, + "measured_bytes": 310246072, + "total_bytes": 310246072, + "observations": [ + { + "id": 8522777408, + "key": "gha-obs-xcode-cadence-20261005-obs-studio-xcode-macOS-ARM64-37322621030-1", + "ref": "refs/heads/cadence-rehearsal-rolling", + "version": "cadf36d522659e940b6cc802fa2ca1da67be9f45499a4ac6d0f8fcb5d0b36fe4", + "created_at": "2026-10-05T14:22:57.114691Z", + "last_accessed_at": "2026-10-05T14:26:07.297970Z", + "size_in_bytes": 310246072 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "04d8967656361ab4b90ccaeec1c591f9c024449d" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323234591", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174567", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37323234591-1", + "series": { + "id": "cadence-rolling-advance-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323234591" + ] +} diff --git a/results/obs-studio/cadence-rolling-advance-xcode/series.json b/results/obs-studio/cadence-rolling-advance-xcode/series.json new file mode 100644 index 000000000..62f430792 --- /dev/null +++ b/results/obs-studio/cadence-rolling-advance-xcode/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-advance-xcode", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "215aea66ef6c9adae46be8c8f6d89395e0ffdffcb64b7184ef9f0157512506cc", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "97ca37291c4e5f5c5c8a0719b8c7b897d2fbc6b5", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "04d8967656361ab4b90ccaeec1c591f9c024449d", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "c42ea925dc2d15cb5a24ef14e27b267001b6bf8b9ed29989e51c7e927e3875e4" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json b/results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json new file mode 100644 index 000000000..6c37ccd69 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/completions/37323465858.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache-02", + "sample": 1, + "run_id": "37323465858", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858", + "source_sha": "c19b136d6863c141212af5aa5620750e485f3009", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "aa2fa4b21b48ff7c23352fecee8574cde35af7f040add1063089e2906d2fe45e" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..0df92f38c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache", + "series_id": "cadence-rolling-seed-ccache-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:17:07Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323465858, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/report.json b/results/obs-studio/cadence-rolling-seed-ccache-02/report.json new file mode 100644 index 000000000..e7cc2fcc8 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/report.json @@ -0,0 +1,470 @@ +{ + "schema_version": 1, + "plan_sha256": "5340d6e1868ad898bd6953d3231a6971cf4ea0fb838c12181a663edbdd6191b2", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache-02", + "sample": 1, + "run_id": "37323465858", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323465858", + "source_sha": "c19b136d6863c141212af5aa5620750e485f3009", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "aa2fa4b21b48ff7c23352fecee8574cde35af7f040add1063089e2906d2fe45e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 644.054199763, + "min": 644.054199763, + "max": 644.054199763 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 631.7126567849999, + "min": 631.7126567849999, + "max": 631.7126567849999 + }, + "storage": { + "median": 172201371, + "min": 172201371, + "max": 172201371 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 643.054199763, + "total_seconds": 644.054199763, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 644.054199763 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174504", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 629.7126567849999, + "total_seconds": 631.7126567849999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 631.7126567849999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 172201371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 172201371, + "unmeasured_tags": [], + "total_bytes": 172201371, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 172201371, + "compressed_size": 172201371, + "size": 172201371 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174505", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/report.md b/results/obs-studio/cadence-rolling-seed-ccache-02/report.md new file mode 100644 index 000000000..0548dbb09 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-seed-ccache-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 644.054199763 | 644.054199763–644.054199763 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 631.7126567849999 | 631.7126567849999–631.7126567849999 | 172201371 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 1 | 643.054199763 | 644.054199763 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 2 | 629.7126567849999 | 631.7126567849999 | 172201371 | boringcache-check | miss | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..f4287cda3 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 643.054199763, + "total_seconds": 644.054199763, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 644.054199763 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174504", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..9c89fe397 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 629.7126567849999, + "total_seconds": 631.7126567849999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 631.7126567849999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 172201371, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "complete": true, + "measured_bytes": 172201371, + "unmeasured_tags": [], + "total_bytes": 172201371, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "tag": "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 172201371, + "compressed_size": 172201371, + "size": 172201371 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "ccache", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-ccache-v2-ccache-ubuntu-26-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323465858", + "run_attempt": "1", + "job": "boringcache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174505", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37323465858-1", + "series": { + "id": "cadence-rolling-seed-ccache-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323465858" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache-02/series.json b/results/obs-studio/cadence-rolling-seed-ccache-02/series.json new file mode 100644 index 000000000..c70a838a7 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache-02", + "created_at": "2026-10-05T14:17:04Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "2e9ad548c2838d146f384eacff6dfd5d35236b59e9005b2d9f78f4db0eda8b8a", + "variant": "ccache", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache-v2", + "cache_tool": "ccache" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "5340d6e1868ad898bd6953d3231a6971cf4ea0fb838c12181a663edbdd6191b2" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json b/results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json new file mode 100644 index 000000000..bbc958a7c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/completions/37322616359.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "sample": 1, + "run_id": "37322616359", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio ccache commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "5320b399ed49d4a3e415fe897b8181b80e68beb26a4ffe76ba463c02f808421c" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..35f32d85c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache", + "cache_tool": "ccache", + "series_id": "cadence-rolling-seed-ccache", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:10:46Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322616359, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359" +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/report.json b/results/obs-studio/cadence-rolling-seed-ccache/report.json new file mode 100644 index 000000000..79073cf9e --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/report.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "plan_sha256": "bd29f1e4edeb3be507ce538fa07532051b53d8d111665c7717bc6121e00d8657", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "sample": 1, + "run_id": "37322616359", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio ccache commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "5320b399ed49d4a3e415fe897b8181b80e68beb26a4ffe76ba463c02f808421c" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "sample": 1, + "run_id": "37322616359", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322616359", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio ccache commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "5320b399ed49d4a3e415fe897b8181b80e68beb26a4ffe76ba463c02f808421c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 577.6759015939999, + "min": 577.6759015939999, + "max": 577.6759015939999 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 577.6759015939999, + "total_seconds": 577.6759015939999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 577.6759015939999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322616359", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174437", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37322616359-1", + "series": { + "id": "cadence-rolling-seed-ccache", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322616359" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/report.md b/results/obs-studio/cadence-rolling-seed-ccache/report.md new file mode 100644 index 000000000..efa10be9f --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/report.md @@ -0,0 +1,37 @@ +# obs-studio: cadence-rolling-seed-ccache + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 577.6759015939999 | 577.6759015939999–577.6759015939999 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +Run 37322616359 failed completion checks: + +- Workflow concluded failure +- BoringCache obs-studio ccache commit: failure +- obs-studio rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 577.6759015939999 | 577.6759015939999 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..5be1b4f74 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "ccache", + "mode": "ccache", + "adapter": "ccache", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 577.6759015939999, + "total_seconds": 577.6759015939999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 577.6759015939999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322616359", + "run_attempt": "1", + "job": "actions-cache-ccache", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174437", + "runner_environment": "github-hosted", + "runner_image": "ubuntu26", + "runner_image_version": "20260927.149.1" + }, + "run_uid": "gh-37322616359-1", + "series": { + "id": "cadence-rolling-seed-ccache", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu26", + "image_version": "20260927.149.1", + "machine": "ubuntu-26.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322616359" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-ccache/series.json b/results/obs-studio/cadence-rolling-seed-ccache/series.json new file mode 100644 index 000000000..cb4e0e04a --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-ccache/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-ccache", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "variant": "ccache", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-ccache", + "cache_tool": "ccache" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "bd29f1e4edeb3be507ce538fa07532051b53d8d111665c7717bc6121e00d8657" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json b/results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json new file mode 100644 index 000000000..2e44dcfc5 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/completions/37326133177.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode-02", + "sample": 1, + "run_id": "37326133177", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177", + "source_sha": "e6e7a3da41e0e94deaae9dbcb4ed530aaa3c59f1", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1494eea0f15a03a25bb4547ac541b315520ae761dc4a1cd57d90d9a731899297" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..6e06f0b7a --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode", + "series_id": "cadence-rolling-seed-xcode-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:36:58Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37326133177, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/report.json b/results/obs-studio/cadence-rolling-seed-xcode-02/report.json new file mode 100644 index 000000000..f34ae8ca8 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/report.json @@ -0,0 +1,470 @@ +{ + "schema_version": 1, + "plan_sha256": "37406699c5188bce11315d38f171cad90fec8af90d48603b103699b9dc475c84", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode-02", + "sample": 1, + "run_id": "37326133177", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37326133177", + "source_sha": "e6e7a3da41e0e94deaae9dbcb4ed530aaa3c59f1", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1494eea0f15a03a25bb4547ac541b315520ae761dc4a1cd57d90d9a731899297" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 710.543963, + "min": 710.543963, + "max": 710.543963 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 528.281434, + "min": 528.281434, + "max": 528.281434 + }, + "storage": { + "median": 983997286, + "min": 983997286, + "max": 983997286 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 710.543963, + "total_seconds": 710.543963, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 710.543963 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174629", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] + }, + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 523.281434, + "total_seconds": 528.281434, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 528.281434 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 983997286, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 983997286, + "unmeasured_tags": [], + "total_bytes": 983997286, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 983997286, + "compressed_size": 983997286, + "size": 983997286 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174630", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/report.md b/results/obs-studio/cadence-rolling-seed-xcode-02/report.md new file mode 100644 index 000000000..5ef19aa96 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/report.md @@ -0,0 +1,29 @@ +# obs-studio: cadence-rolling-seed-xcode-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 710.543963 | 710.543963–710.543963 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 528.281434 | 528.281434–528.281434 | 983997286 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 710.543963 | 710.543963 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 5 | 523.281434 | 528.281434 | 983997286 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..8009e6252 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 710.543963, + "total_seconds": 710.543963, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 710.543963 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174629", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..3336d886c --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/runs/1-boringcache-commit.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 523.281434, + "total_seconds": 528.281434, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 528.281434 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode", + "workspace": "boringcache/benchmarks", + "storage_bytes": 983997286, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "complete": true, + "measured_bytes": 983997286, + "unmeasured_tags": [], + "total_bytes": 983997286, + "observations": [ + { + "requested_tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "tag": "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 983997286, + "compressed_size": 983997286, + "size": 983997286 + } + ] + } + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "xcode", + "resolved_tags": [ + "obs-studio-cadence-20261005-obs-studio-xcode-v2-xcode-macos-26-arm64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37326133177", + "run_attempt": "1", + "job": "boringcache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174630", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37326133177-1", + "series": { + "id": "cadence-rolling-seed-xcode-02", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37326133177" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode-02/series.json b/results/obs-studio/cadence-rolling-seed-xcode-02/series.json new file mode 100644 index 000000000..b97f5844b --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode-02/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode-02", + "created_at": "2026-10-05T14:36:56Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e3cfc74b35efcb12b0c07da0072bd267d67d6fe0316de6784c458e5ea2eca6ba", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode-v2", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "37406699c5188bce11315d38f171cad90fec8af90d48603b103699b9dc475c84" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json b/results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json new file mode 100644 index 000000000..4562bc2f6 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/completions/37322621030.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "sample": 1, + "run_id": "37322621030", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "167a4879396a537d636434a85dc880174b79075432c1f84b3bc26955594fbd02" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json b/results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json new file mode 100644 index 000000000..0fd58a6a0 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/dispatches/1-obs-rolling-benchmark.yml.json @@ -0,0 +1,42 @@ +{ + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode", + "series_id": "cadence-rolling-seed-xcode", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "ccache", + "xcode" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:10:49Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322621030, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030" +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/report.json b/results/obs-studio/cadence-rolling-seed-xcode/report.json new file mode 100644 index 000000000..7a262f88a --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/report.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "plan_sha256": "152102f6ca51dcfd61c214b07455c9a2fccc8aba13dc4ce089c3a6b016ca52a9", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "sample": 1, + "run_id": "37322621030", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "167a4879396a537d636434a85dc880174b79075432c1f84b3bc26955594fbd02" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "sample": 1, + "run_id": "37322621030", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322621030", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache obs-studio xcode commit: failure", + "obs-studio rolling report: failure" + ], + "evidence_manifest_sha256": "167a4879396a537d636434a85dc880174b79075432c1f84b3bc26955594fbd02" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 550.518804, + "min": 550.518804, + "max": 550.518804 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 550.518804, + "total_seconds": 550.518804, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 550.518804 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322621030", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174440", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37322621030-1", + "series": { + "id": "cadence-rolling-seed-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322621030" + ] + } + ], + "exclusions": [] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/report.md b/results/obs-studio/cadence-rolling-seed-xcode/report.md new file mode 100644 index 000000000..532eba981 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/report.md @@ -0,0 +1,37 @@ +# obs-studio: cadence-rolling-seed-xcode + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 550.518804 | 550.518804–550.518804 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +Run 37322621030 failed completion checks: + +- Workflow concluded failure +- BoringCache obs-studio xcode commit: failure +- obs-studio rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 0 | 550.518804 | 550.518804 | unmeasured | unmeasured | miss | [JSON](runs/1-actions-cache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json b/results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..eb36c893f --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/runs/1-actions-cache-commit.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "benchmark": "obs-studio", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "xcode", + "mode": "xcode", + "adapter": "xcode", + "case": { + "schema_version": 1, + "case_id": "obs-studio", + "workspace": "boringcache/benchmarks", + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-obs-studio", + "branch": "main", + "revision": "74322882037cad76d0fda72b5ba6058ded815343" + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/obs-studio-obs-actions-cache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-boringcache.yml", + "variant_input": "cache_tool", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "ccache", + "xcode" + ] + }, + { + "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": {}, + "variants": [ + "ccache", + "xcode" + ] + } + ], + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, + "sync_minute": 18, + "source_prefix": "OBS" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 550.518804, + "total_seconds": 550.518804, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "build_and_reuse_seconds": 550.518804 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "obsproject/obs-studio", + "sha": "ef5814431c6af9061874470201d0b799bc84602b" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322621030", + "run_attempt": "1", + "job": "actions-cache-xcode", + "workflow": "OBS Studio comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "macOS", + "runner_arch": "ARM64", + "runner_name": "GitHub Actions 1000174440", + "runner_environment": "github-hosted", + "runner_image": "macos26", + "runner_image_version": "20260907.0351.1" + }, + "run_uid": "gh-37322621030-1", + "series": { + "id": "cadence-rolling-seed-xcode", + "sample": 1 + }, + "environment": { + "os": "macOS", + "arch": "ARM64", + "image": "macos26", + "image_version": "20260907.0351.1", + "machine": "macos-26" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322621030" + ] +} diff --git a/results/obs-studio/cadence-rolling-seed-xcode/series.json b/results/obs-studio/cadence-rolling-seed-xcode/series.json new file mode 100644 index 000000000..87f6e29c9 --- /dev/null +++ b/results/obs-studio/cadence-rolling-seed-xcode/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "cadence-rolling-seed-xcode", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "971d068999fdb986de97e99f67123b94fd3c4fccc1bdeda4ed454ccacb7b0ec3", + "variant": "xcode", + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-obs-studio-xcode", + "cache_tool": "xcode" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "obsproject/obs-studio", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" + } + ] + }, + "verification": [ + "Pinned upstream recipe and commands", + "Successful upstream build", + "Native ccache or Xcode output verification for reuse observations" + ], + "plan_sha256": "152102f6ca51dcfd61c214b07455c9a2fccc8aba13dc4ce089c3a6b016ca52a9" +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json b/results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json new file mode 100644 index 000000000..c85407c6f --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/completions/37324223748.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324223748", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "4834dc48812d1ab6f8e6114630757a20ed7661c1b4d1493705efe89e8b3d00b6" +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..af63d983c --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:47Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324223748, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748" +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/report.json b/results/opencut-moon/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..d84d7dc86 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/report.json @@ -0,0 +1,220 @@ +{ + "schema_version": 1, + "plan_sha256": "78b7796942758265e9ac7bab7cce6a7a579307fdafc10566522875719f33976d", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324223748", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "4834dc48812d1ab6f8e6114630757a20ed7661c1b4d1493705efe89e8b3d00b6" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324223748", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324223748", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "4834dc48812d1ab6f8e6114630757a20ed7661c1b4d1493705efe89e8b3d00b6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10101780600000154, + "build_seconds": 9.075982147000005, + "workflow_seconds": null, + "total_seconds": 9.176999953000006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0015317619999990484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.176999953000006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324223748", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174547", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324223748-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324223748" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/report.md b/results/opencut-moon/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..7380ccac5 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/report.md @@ -0,0 +1,38 @@ +# opencut-moon: cadence-rolling-advance-02 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37324223748 failed completion checks: + +- Workflow concluded failure +- BoringCache opencut-moon commit: failure +- opencut-moon rolling report: failure + +Missing observations: + +- Sample 1, BoringCache, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10101780600000154 | 9.075982147000005 | 9.176999953000006 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..147650935 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10101780600000154, + "build_seconds": 9.075982147000005, + "workflow_seconds": null, + "total_seconds": 9.176999953000006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0015317619999990484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.176999953000006 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324223748", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174547", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324223748-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324223748" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance-02/series.json b/results/opencut-moon/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..b091032e7 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "78b7796942758265e9ac7bab7cce6a7a579307fdafc10566522875719f33976d" +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json b/results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json new file mode 100644 index 000000000..5de34bffb --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/completions/37325287254.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325287254", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "b444b24fbab3a1d42d2449737aa46e619bf28e4a2dfc85f3daab79449e796e0c" +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..cd66b61d2 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3", + "series_id": "cadence-rolling-advance-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:30:41Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325287254, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254" +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/report.json b/results/opencut-moon/cadence-rolling-advance-03/report.json new file mode 100644 index 000000000..fc6b8ff13 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "583701a786de16d224f09043f85430491ec6c9f58fdd9b4431b984561cc22f3a", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-03", + "sample": 1, + "run_id": "37325287254", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325287254", + "source_sha": "e66f36d29d96f43ed3eb2e0490cef18ef5bb921c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "b444b24fbab3a1d42d2449737aa46e619bf28e4a2dfc85f3daab79449e796e0c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108759999999961, + "build_seconds": 13.598188859999993, + "workflow_seconds": null, + "total_seconds": 13.699276459999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0017177939999868386, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.699276459999993 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174602", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20152445499999772, + "build_seconds": 14.694388483000012, + "workflow_seconds": null, + "total_seconds": 14.89591293800001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.35811930500000244, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.89591293800001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174603", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/report.md b/results/opencut-moon/cadence-rolling-advance-03/report.md new file mode 100644 index 000000000..50e748f52 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-advance-03 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10108759999999961 | 13.598188859999993 | 13.699276459999993 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20152445499999772 | 14.694388483000012 | 14.89591293800001 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..62c62dbdc --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108759999999961, + "build_seconds": 13.598188859999993, + "workflow_seconds": null, + "total_seconds": 13.699276459999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0017177939999868386, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.699276459999993 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174602", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..7b0554019 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20152445499999772, + "build_seconds": 14.694388483000012, + "workflow_seconds": null, + "total_seconds": 14.89591293800001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.35811930500000244, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.89591293800001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325287254", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174603", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325287254-1", + "series": { + "id": "cadence-rolling-advance-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325287254" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance-03/series.json b/results/opencut-moon/cadence-rolling-advance-03/series.json new file mode 100644 index 000000000..bfc2814ff --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance-03", + "created_at": "2026-10-05T14:30:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "66deccb1e54523e9fe2e6912379bfb779717bccb1e78f566a84b2dc7095d4b2f", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "583701a786de16d224f09043f85430491ec6c9f58fdd9b4431b984561cc22f3a" +} diff --git a/results/opencut-moon/cadence-rolling-advance/completions/37323159608.json b/results/opencut-moon/cadence-rolling-advance/completions/37323159608.json new file mode 100644 index 000000000..a3adbdc91 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/completions/37323159608.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323159608", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "86eaebf0d054dc81b425ce0ae87c8bbce0cc185b4bad17ea13dc7b2e1ef1fd20" +} diff --git a/results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..f37153388 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:52Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323159608, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608" +} diff --git a/results/opencut-moon/cadence-rolling-advance/report.json b/results/opencut-moon/cadence-rolling-advance/report.json new file mode 100644 index 000000000..bb2a2ceeb --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "b0cbc49ae9e5b938a1a5379204ddc56b2d1349af8b6ff6452f4ab4790776ebfe", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323159608", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "86eaebf0d054dc81b425ce0ae87c8bbce0cc185b4bad17ea13dc7b2e1ef1fd20" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323159608", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323159608", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote opencut-moon commit: failure", + "opencut-moon rolling report: failure" + ], + "evidence_manifest_sha256": "86eaebf0d054dc81b425ce0ae87c8bbce0cc185b4bad17ea13dc7b2e1ef1fd20" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7c8bfdbc420b4d1e9e9edfd49359f619498d3783a60d1e01da270448115f7553", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.3016585379999981, + "build_seconds": 11.28177397799999, + "workflow_seconds": null, + "total_seconds": 11.583432515999988, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4176102280000009, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 11.583432515999988 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323159608", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174486", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323159608-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323159608" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-advance/report.md b/results/opencut-moon/cadence-rolling-advance/report.md new file mode 100644 index 000000000..bb9232332 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# opencut-moon: cadence-rolling-advance + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323159608 failed completion checks: + +- Workflow concluded failure +- bazel-remote opencut-moon commit: failure +- opencut-moon rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.3016585379999981 | 11.28177397799999 | 11.583432515999988 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..506d5cf95 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7c8bfdbc420b4d1e9e9edfd49359f619498d3783a60d1e01da270448115f7553", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.3016585379999981, + "build_seconds": 11.28177397799999, + "workflow_seconds": null, + "total_seconds": 11.583432515999988, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.4176102280000009, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 11.583432515999988 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323159608", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174486", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323159608-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323159608" + ] +} diff --git a/results/opencut-moon/cadence-rolling-advance/series.json b/results/opencut-moon/cadence-rolling-advance/series.json new file mode 100644 index 000000000..efe4bd12e --- /dev/null +++ b/results/opencut-moon/cadence-rolling-advance/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "7c8bfdbc420b4d1e9e9edfd49359f619498d3783a60d1e01da270448115f7553", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b0cbc49ae9e5b938a1a5379204ddc56b2d1349af8b6ff6452f4ab4790776ebfe" +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json b/results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json new file mode 100644 index 000000000..0344e8d5f --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/completions/37324020999.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324020999", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324020999", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "52e63f033403920d3155d562b3e02d8d1e7c85fb46197895ef027dee1237cd9c" +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..a5091a107 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:17Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324020999, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324020999" +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/report.json b/results/opencut-moon/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..ea13da69d --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "a8ca6ebc122865dc73ab748ade79bf221a2a2c16c5e1fbd86e9e7fb465e4fc9f", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324020999", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324020999", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "52e63f033403920d3155d562b3e02d8d1e7c85fb46197895ef027dee1237cd9c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10247405000001208, + "build_seconds": 15.200883040999997, + "workflow_seconds": null, + "total_seconds": 15.30335709100001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013556339999922784, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.30335709100001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174528", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011381649999521, + "build_seconds": 16.51002464100003, + "workflow_seconds": null, + "total_seconds": 16.61116280599998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.45349295599999095, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.61116280599998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174529", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/report.md b/results/opencut-moon/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..e29c5b0e2 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-seed-02 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10247405000001208 | 15.200883040999997 | 15.30335709100001 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.1011381649999521 | 16.51002464100003 | 16.61116280599998 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..43e3f766b --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10247405000001208, + "build_seconds": 15.200883040999997, + "workflow_seconds": null, + "total_seconds": 15.30335709100001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013556339999922784, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.30335709100001 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174528", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..445552c2e --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011381649999521, + "build_seconds": 16.51002464100003, + "workflow_seconds": null, + "total_seconds": 16.61116280599998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.45349295599999095, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.61116280599998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324020999", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174529", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324020999-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324020999" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-02/series.json b/results/opencut-moon/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..d26d72f17 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v2" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "a8ca6ebc122865dc73ab748ade79bf221a2a2c16c5e1fbd86e9e7fb465e4fc9f" +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json b/results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json new file mode 100644 index 000000000..88e47abb0 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/completions/37325140807.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325140807", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ab7c1d13b79fedd9c254e3d5d75ef4b45a857dd5f507e1a35d474d6a9faa7b7e" +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..39b3845a7 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3", + "series_id": "cadence-rolling-seed-03", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:29:36Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37325140807, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807" +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/report.json b/results/opencut-moon/cadence-rolling-seed-03/report.json new file mode 100644 index 000000000..acdd81f7c --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "c6e4c83124eebe212e820a4b00cbd5cd89edd24af9e660b4cf54e852544a2d78", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-03", + "sample": 1, + "run_id": "37325140807", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325140807", + "source_sha": "fbbb87e90b95bee5e0821ca9e54cad5f4ab014fb", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ab7c1d13b79fedd9c254e3d5d75ef4b45a857dd5f507e1a35d474d6a9faa7b7e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.101062933999998, + "build_seconds": 14.702613067999998, + "workflow_seconds": null, + "total_seconds": 14.803676001999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016717000000028293, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.803676001999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174593", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014604100000028, + "build_seconds": 14.595511775999995, + "workflow_seconds": null, + "total_seconds": 14.796972185999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.1871384540000065, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.796972185999998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174592", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/report.md b/results/opencut-moon/cadence-rolling-seed-03/report.md new file mode 100644 index 000000000..71ab1926f --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-seed-03 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.101062933999998 | 14.702613067999998 | 14.803676001999996 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.2014604100000028 | 14.595511775999995 | 14.796972185999998 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..bdc8abcd8 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.101062933999998, + "build_seconds": 14.702613067999998, + "workflow_seconds": null, + "total_seconds": 14.803676001999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016717000000028293, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.803676001999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174593", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json new file mode 100644 index 000000000..85fa0fa29 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014604100000028, + "build_seconds": 14.595511775999995, + "workflow_seconds": null, + "total_seconds": 14.796972185999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.1871384540000065, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 14.796972185999998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon-v3", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325140807", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174592", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325140807-1", + "series": { + "id": "cadence-rolling-seed-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325140807" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed-03/series.json b/results/opencut-moon/cadence-rolling-seed-03/series.json new file mode 100644 index 000000000..51de22bf5 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed-03", + "created_at": "2026-10-05T14:29:29Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "086d143d15b8374ed27463ab5f3bc8474ad07544017f378f25a3d8e69b7b680e", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon-v3" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "c6e4c83124eebe212e820a4b00cbd5cd89edd24af9e660b4cf54e852544a2d78" +} diff --git a/results/opencut-moon/cadence-rolling-seed/completions/37322625973.json b/results/opencut-moon/cadence-rolling-seed/completions/37322625973.json new file mode 100644 index 000000000..19b8d39f9 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/completions/37322625973.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322625973", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322625973", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7dd25aa5714d0dfeef7251b47ca48503eed5cc091bc592fa91b24ed7ede23a1c" +} diff --git a/results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..323260711 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:51Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322625973, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322625973" +} diff --git a/results/opencut-moon/cadence-rolling-seed/report.json b/results/opencut-moon/cadence-rolling-seed/report.json new file mode 100644 index 000000000..bb008883d --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "4306a91ea3011928b3bc572cd3c97705532f20966ab26f7a98a287b5c94abb67", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322625973", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322625973", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7dd25aa5714d0dfeef7251b47ca48503eed5cc091bc592fa91b24ed7ede23a1c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009767549999907, + "build_seconds": 15.103258492999984, + "workflow_seconds": null, + "total_seconds": 15.204235247999975, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001731152000019165, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.204235247999975 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174446", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20123272899999733, + "build_seconds": 16.601319771000007, + "workflow_seconds": null, + "total_seconds": 16.802552500000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.310330332999996, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.802552500000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174447", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/cadence-rolling-seed/report.md b/results/opencut-moon/cadence-rolling-seed/report.md new file mode 100644 index 000000000..ce5a080a8 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# opencut-moon: cadence-rolling-seed + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.1009767549999907 | 15.103258492999984 | 15.204235247999975 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20123272899999733 | 16.601319771000007 | 16.802552500000004 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..35a574729 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009767549999907, + "build_seconds": 15.103258492999984, + "workflow_seconds": null, + "total_seconds": 15.204235247999975, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001731152000019165, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 15.204235247999975 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174446", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..d3e87da84 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20123272899999733, + "build_seconds": 16.601319771000007, + "workflow_seconds": null, + "total_seconds": 16.802552500000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.310330332999996, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 16.802552500000004 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-rolling-cadence-20261005-opencut-moon", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "400f097becba5db0fbc305d5a65348cb81c20356" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322625973", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174447", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322625973-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322625973" + ] +} diff --git a/results/opencut-moon/cadence-rolling-seed/series.json b/results/opencut-moon/cadence-rolling-seed/series.json new file mode 100644 index 000000000..9fa7665b4 --- /dev/null +++ b/results/opencut-moon/cadence-rolling-seed/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "2f06609e7a6179df13201f2d21f9c0cb87257d1d7884d27e133e7eea2f048ad4", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-opencut-moon" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "400f097becba5db0fbc305d5a65348cb81c20356" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "4306a91ea3011928b3bc572cd3c97705532f20966ab26f7a98a287b5c94abb67" +} diff --git a/results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json b/results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..0224a1f26 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers", + "series_id": "cadence-rolling-advance-layers", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:14:53Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323164071, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071" +} diff --git a/results/posthog/cadence-rolling-advance-layers/series.json b/results/posthog/cadence-rolling-advance-layers/series.json new file mode 100644 index 000000000..c34a1df67 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-advance-layers", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "9c9f99841fba0c7a6826bcd1d39d5d5268e5aa213c5e838ed0bf08b4e7d23ecb" +} diff --git a/results/posthog/cadence-rolling-seed-layers/completions/37321772493.json b/results/posthog/cadence-rolling-seed-layers/completions/37321772493.json new file mode 100644 index 000000000..d81853290 --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/completions/37321772493.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-seed-layers", + "sample": 1, + "run_id": "37321772493", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "663ed2039ee1f1f48a2cf710d82909d9f3b7530f78dc1fea94b2a5e984134623" +} diff --git a/results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json b/results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json new file mode 100644 index 000000000..122333a33 --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/dispatches/1-native-rolling-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers", + "series_id": "cadence-rolling-seed-layers", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-05T14:04:30Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37321772493, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493" +} diff --git a/results/posthog/cadence-rolling-seed-layers/report.json b/results/posthog/cadence-rolling-seed-layers/report.json new file mode 100644 index 000000000..7fee23e4b --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/report.json @@ -0,0 +1,509 @@ +{ + "schema_version": 1, + "plan_sha256": "4ffb50f0d4f2613dc4e123d6d90cdbef454111c9a523e945c745d094180bdd7c", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-seed-layers", + "sample": 1, + "run_id": "37321772493", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37321772493", + "source_sha": "410dd8bbf3d13a5a4dbe14acb9d66d1923c3d361", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "663ed2039ee1f1f48a2cf710d82909d9f3b7530f78dc1fea94b2a5e984134623" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 2088, + "min": 2088, + "max": 2088 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 814, + "min": 814, + "max": 814 + }, + "storage": { + "median": 6243631290, + "min": 6243631290, + "max": 6243631290 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 2077, + "workflow_seconds": 2134, + "total_seconds": 2088, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2088 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174366", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 814, + "workflow_seconds": 832, + "total_seconds": 814, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 814 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6243631290, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6243631290, + "unmeasured_tags": [], + "total_bytes": 6243631290, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b9df360b-3d34-4c58-9153-9689fc78eccb", + "manifest_root_digest": "sha256:65d4fc042f3ae490b0a4ab7d39f9d28283149ce7afb17582bcd8618e563e3b41", + "kv_total_size": 0, + "compressed_size": 6243631290, + "size": 6243631290 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174367", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/cadence-rolling-seed-layers/report.md b/results/posthog/cadence-rolling-seed-layers/report.md new file mode 100644 index 000000000..95c24340d --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/report.md @@ -0,0 +1,29 @@ +# posthog: cadence-rolling-seed-layers + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 2088 | 2088–2088 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 814 | 814–814 | 6243631290 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 11 | 2077 | 2088 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 814 | 814 | 6243631290 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json b/results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..475f7d27f --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/runs/1-actions-cache-commit.json @@ -0,0 +1,199 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 2077, + "workflow_seconds": 2134, + "total_seconds": 2088, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2088 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174366", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] +} diff --git a/results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json b/results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f27205b6a --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/runs/1-boringcache-commit.json @@ -0,0 +1,245 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 814, + "workflow_seconds": 832, + "total_seconds": 814, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 814 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6243631290, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6243631290, + "unmeasured_tags": [], + "total_bytes": 6243631290, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b9df360b-3d34-4c58-9153-9689fc78eccb", + "manifest_root_digest": "sha256:65d4fc042f3ae490b0a4ab7d39f9d28283149ce7afb17582bcd8618e563e3b41", + "kv_total_size": 0, + "compressed_size": 6243631290, + "size": 6243631290 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37321772493", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174367", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37321772493-1", + "series": { + "id": "cadence-rolling-seed-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37321772493" + ] +} diff --git a/results/posthog/cadence-rolling-seed-layers/series.json b/results/posthog/cadence-rolling-seed-layers/series.json new file mode 100644 index 000000000..e7a16d3cf --- /dev/null +++ b/results/posthog/cadence-rolling-seed-layers/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-seed-layers", + "created_at": "2026-10-05T14:03:52Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "289ee8b1b1fbcaeff6266cbf836ecff45a9b74af3d850d3c335c54c1b9be7e74", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-posthog-layers", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "4ffb50f0d4f2613dc4e123d6d90cdbef454111c9a523e945c745d094180bdd7c" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json b/results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json new file mode 100644 index 000000000..09948412e --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/completions/37324228460.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324228460", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c93fe2621f6909406e9f055215f2c55d6346b2927045b0d52c076c4f404ca55" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..29c47d561 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "series_id": "cadence-rolling-advance-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:22:50Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324228460, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/report.json b/results/stackstorm-pants/cadence-rolling-advance-02/report.json new file mode 100644 index 000000000..8e1f00171 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "ea0b66d0766668904cd63dc76ff37096b632076ea4e27769912bf66353928a78", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance-02", + "sample": 1, + "run_id": "37324228460", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324228460", + "source_sha": "727afc60de757e1e7d7690ff5c9433f06fde54a6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "1c93fe2621f6909406e9f055215f2c55d6346b2927045b0d52c076c4f404ca55" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10122090600000888, + "build_seconds": 13.451143174999999, + "workflow_seconds": null, + "total_seconds": 13.552364081000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0018756030000020019, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.552364081000007 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174561", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10115972500000225, + "build_seconds": 13.277457289000012, + "workflow_seconds": null, + "total_seconds": 13.378617014000014, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.2545980109999846, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.378617014000014 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174562", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/report.md b/results/stackstorm-pants/cadence-rolling-advance-02/report.md new file mode 100644 index 000000000..92d08b5c5 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/report.md @@ -0,0 +1,30 @@ +# stackstorm-pants: cadence-rolling-advance-02 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10122090600000888 | 13.451143174999999 | 13.552364081000007 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.10115972500000225 | 13.277457289000012 | 13.378617014000014 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..9465eddab --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10122090600000888, + "build_seconds": 13.451143174999999, + "workflow_seconds": null, + "total_seconds": 13.552364081000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0018756030000020019, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.552364081000007 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174561", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..7f3ae1445 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10115972500000225, + "build_seconds": 13.277457289000012, + "workflow_seconds": null, + "total_seconds": 13.378617014000014, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.2545980109999846, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 13.378617014000014 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324228460", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174562", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324228460-1", + "series": { + "id": "cadence-rolling-advance-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324228460" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance-02/series.json b/results/stackstorm-pants/cadence-rolling-advance-02/series.json new file mode 100644 index 000000000..ac5f85e39 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance-02", + "created_at": "2026-10-05T14:22:36Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "1099cc3accb93f83606c58da08608d17994c4574865cd2e910b44f186f0cce8e", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "ea0b66d0766668904cd63dc76ff37096b632076ea4e27769912bf66353928a78" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json b/results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json new file mode 100644 index 000000000..d4e0605c6 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/completions/37323169828.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323169828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote stackstorm-pants commit: failure", + "stackstorm-pants rolling report: failure" + ], + "evidence_manifest_sha256": "ec94c08b8d5b64fa278e36cb795b9fac3c01f887411aeada029a317606e965de" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..d901a92c4 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:55Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323169828, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828" +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/report.json b/results/stackstorm-pants/cadence-rolling-advance/report.json new file mode 100644 index 000000000..08611b3e9 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "cbd4ab69f17a3ca86a1858a9378069d9fe3b8a14bdfe725bc465c02b03b44f5f", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323169828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote stackstorm-pants commit: failure", + "stackstorm-pants rolling report: failure" + ], + "evidence_manifest_sha256": "ec94c08b8d5b64fa278e36cb795b9fac3c01f887411aeada029a317606e965de" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323169828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323169828", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote stackstorm-pants commit: failure", + "stackstorm-pants rolling report: failure" + ], + "evidence_manifest_sha256": "ec94c08b8d5b64fa278e36cb795b9fac3c01f887411aeada029a317606e965de" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "commit", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1f27ac2b7ec3728c03cc60e99170085d91fe64ae531425a6ef555a8045377542", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.14576035199999637, + "build_seconds": 9.821776771000003, + "workflow_seconds": null, + "total_seconds": 9.967537123, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.36232383699999104, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.967537123 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323169828", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174491", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323169828-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323169828" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/report.md b/results/stackstorm-pants/cadence-rolling-advance/report.md new file mode 100644 index 000000000..1994e1ba1 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/report.md @@ -0,0 +1,38 @@ +# stackstorm-pants: cadence-rolling-advance + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 1/2 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +Run 37323169828 failed completion checks: + +- Workflow concluded failure +- bazel-remote stackstorm-pants commit: failure +- stackstorm-pants rolling report: failure + +Missing observations: + +- Sample 1, bazel-remote, Changed-source build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 0.14576035199999637 | 9.821776771000003 | 9.967537123 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..5c330abc9 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1f27ac2b7ec3728c03cc60e99170085d91fe64ae531425a6ef555a8045377542", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.14576035199999637, + "build_seconds": 9.821776771000003, + "workflow_seconds": null, + "total_seconds": 9.967537123, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.36232383699999104, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 9.967537123 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323169828", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174491", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323169828-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323169828" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-advance/series.json b/results/stackstorm-pants/cadence-rolling-advance/series.json new file mode 100644 index 000000000..b7c7e73d5 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-advance/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "1f27ac2b7ec3728c03cc60e99170085d91fe64ae531425a6ef555a8045377542", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "cbd4ab69f17a3ca86a1858a9378069d9fe3b8a14bdfe725bc465c02b03b44f5f" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json b/results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json new file mode 100644 index 000000000..708d0d6aa --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/completions/37324025714.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324025714", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "31be84e9a39d3b291d98a29e7b454c5001a2c6c6fae53ce2590634ba0ba9968c" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..38bad27fb --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2", + "series_id": "cadence-rolling-seed-02", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:21:19Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37324025714, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/report.json b/results/stackstorm-pants/cadence-rolling-seed-02/report.json new file mode 100644 index 000000000..8dae7d774 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "dccfdc29d50f268b3dd983b28b87c7941a08d5d6532b8d453b08efde42be3a12", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed-02", + "sample": 1, + "run_id": "37324025714", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37324025714", + "source_sha": "c21e4fb40b666041338169883064b57c9ff8c590", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "31be84e9a39d3b291d98a29e7b454c5001a2c6c6fae53ce2590634ba0ba9968c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10118893100001003, + "build_seconds": 28.556505596999983, + "workflow_seconds": null, + "total_seconds": 28.657694527999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0032910459999868635, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.657694527999993 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174530", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2008419569999944, + "build_seconds": 17.43477562000001, + "workflow_seconds": null, + "total_seconds": 17.635617577000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6462549910000064, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 17.635617577000005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174531", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/report.md b/results/stackstorm-pants/cadence-rolling-seed-02/report.md new file mode 100644 index 000000000..4a87bd6b0 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/report.md @@ -0,0 +1,30 @@ +# stackstorm-pants: cadence-rolling-seed-02 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10118893100001003 | 28.556505596999983 | 28.657694527999993 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.2008419569999944 | 17.43477562000001 | 17.635617577000005 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..3a0acc496 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10118893100001003, + "build_seconds": 28.556505596999983, + "workflow_seconds": null, + "total_seconds": 28.657694527999993, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0032910459999868635, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.657694527999993 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174530", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json new file mode 100644 index 000000000..c8922c266 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2008419569999944, + "build_seconds": 17.43477562000001, + "workflow_seconds": null, + "total_seconds": 17.635617577000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6462549910000064, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 17.635617577000005 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants-v2", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37324025714", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174531", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37324025714-1", + "series": { + "id": "cadence-rolling-seed-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37324025714" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed-02/series.json b/results/stackstorm-pants/cadence-rolling-seed-02/series.json new file mode 100644 index 000000000..4aa0d4b33 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed-02/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed-02", + "created_at": "2026-10-05T14:21:11Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "ec4dfe18ec85772d713bf6cad9afc90df367c92fc02484d7aa3de794977a1d3c", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants-v2" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "dccfdc29d50f268b3dd983b28b87c7941a08d5d6532b8d453b08efde42be3a12" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json b/results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json new file mode 100644 index 000000000..4d9b4d5f8 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/completions/37322630658.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322630658", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322630658", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0f464c2e7d5b9cf85c01cef28c2ef7b82927a419ece833446ecdaeafa9fb59da" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json b/results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json new file mode 100644 index 000000000..8aee191c0 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/dispatches/1-reapi-rolling-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:53Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322630658, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322630658" +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/report.json b/results/stackstorm-pants/cadence-rolling-seed/report.json new file mode 100644 index 000000000..c37577793 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/report.json @@ -0,0 +1,348 @@ +{ + "schema_version": 1, + "plan_sha256": "bc845e5486026ab37d50479ebbfd7a316acb01f5870abd1b0254c58c4612e220", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322630658", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322630658", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0f464c2e7d5b9cf85c01cef28c2ef7b82927a419ece833446ecdaeafa9fb59da" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107338099999197, + "build_seconds": 28.043372379000004, + "workflow_seconds": null, + "total_seconds": 28.144445759999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003087786999998343, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.144445759999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174448", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135293900000306, + "build_seconds": 26.938155466000012, + "workflow_seconds": null, + "total_seconds": 27.139508405000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6489476249999484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.139508405000015 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174449", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/report.md b/results/stackstorm-pants/cadence-rolling-seed/report.md new file mode 100644 index 000000000..ca27a5fc9 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/report.md @@ -0,0 +1,30 @@ +# stackstorm-pants: cadence-rolling-seed + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Changed-source build | bazel-remote | 1 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Changed-source build | 0.10107338099999197 | 28.043372379000004 | 28.144445759999996 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-commit.json) | +| 1 | BoringCache | Changed-source build | 0.20135293900000306 | 26.938155466000012 | 27.139508405000015 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json b/results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json new file mode 100644 index 000000000..6aae4471c --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/runs/1-bazel-remote-commit.json @@ -0,0 +1,146 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107338099999197, + "build_seconds": 28.043372379000004, + "workflow_seconds": null, + "total_seconds": 28.144445759999996, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003087786999998343, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 28.144445759999996 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174448", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..250267a68 --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,148 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135293900000306, + "build_seconds": 26.938155466000012, + "workflow_seconds": null, + "total_seconds": 27.139508405000015, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6489476249999484, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "build_and_reuse_seconds": 27.139508405000015 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-rolling-cadence-20261005-stackstorm-pants", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322630658", + "run_attempt": "1", + "job": "commit", + "workflow": "REAPI comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174449", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322630658-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322630658" + ] +} diff --git a/results/stackstorm-pants/cadence-rolling-seed/series.json b/results/stackstorm-pants/cadence-rolling-seed/series.json new file mode 100644 index 000000000..3f30d586a --- /dev/null +++ b/results/stackstorm-pants/cadence-rolling-seed/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "b6178745cc23a2625f5ccae729fa22293bcf6ea853fd3c0139a067d35007016b", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-stackstorm-pants" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "8c57959f3a23cafc55312e28c5beb7b2dab045a3" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "bc845e5486026ab37d50479ebbfd7a316acb01f5870abd1b0254c58c4612e220" +} diff --git a/results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json b/results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..36725eb6b --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/dispatches/2-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:43:54Z", + "ref": "cadence-fresh-qualification", + "state": "requested", + "run_id": 37327069343, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37327069343" +} diff --git a/results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json b/results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..73f4dc197 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-advance", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:14:58Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37323174641, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641" +} diff --git a/results/zed-nix/cadence-rolling-advance/series.json b/results/zed-nix/cadence-rolling-advance/series.json new file mode 100644 index 000000000..b3b752f74 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-advance", + "created_at": "2026-10-05T14:13:46Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "variant": null, + "workflow_inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "5a98dac1491c4ace612df8a0bfcf5e2c14c0f6c1697f21dd0ddb3c65fcbebef1" +} diff --git a/results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json b/results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json new file mode 100644 index 000000000..f1a1e113a --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/dispatches/1-nix-rolling-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-rolling-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "rolling", + "inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-rolling-seed", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T14:10:56Z", + "ref": "cadence-rehearsal-rolling", + "state": "requested", + "run_id": 37322636399, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399" +} diff --git a/results/zed-nix/cadence-rolling-seed/series.json b/results/zed-nix/cadence-rolling-seed/series.json new file mode 100644 index 000000000..e9d00100d --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-seed", + "created_at": "2026-10-05T14:09:57Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "variant": null, + "workflow_inputs": { + "case_id": "zed-nix", + "cli_version": "vcli-canary-7a5b27146ebe", + "cache_scope": "cadence-20261005-zed-nix", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "bf6c8078a29447c37ad71c86489ed872363058a299dcf47768b2c5e99b7c03af" +} diff --git a/scripts/benchmark-catalog.rb b/scripts/benchmark-catalog.rb index 4ee48d44a..7c09ff4dd 100644 --- a/scripts/benchmark-catalog.rb +++ b/scripts/benchmark-catalog.rb @@ -53,7 +53,8 @@ def self.entry(directory, root:, evidence:) end archives = run_ids.filter_map { |id| evidence[[BenchmarkCases::REPOSITORY, id]] }.map do |archive| archive.slice("repository", "run_id", "inventory_state", "verified_files", "gaps", "scope", - "unavailable_workflow_evidence", "release_url", "bundle_url", "bundle_bytes", "sha256", "publication_verification") + "unavailable_workflow_evidence", "release_url", "bundle_url", "bundle_bytes", "sha256", "publication_verification", + "bundle_format", "archive_sha256", "archive_bytes", "bundle_parts") end base.merge(plan.slice("lane", "variant", "sample_count", "comparison", "source", "plan_sha256", "definition_sha256"), "state" => state, "publication" => report.fetch("publication"), diff --git a/test/benchmark_catalog_test.rb b/test/benchmark_catalog_test.rb index 0f9dc83cd..0bc393317 100644 --- a/test/benchmark_catalog_test.rb +++ b/test/benchmark_catalog_test.rb @@ -97,6 +97,8 @@ def test_failed_completions_and_archive_gaps_remain_visible_without_observations completion(run_id: "12", status: "failed") archive = {"repository" => BenchmarkCases::REPOSITORY, "run_id" => 12, "inventory_state" => "complete", "release_url" => "https://github.com/boringcache/benchmarks/releases/tag/evidence-example", + "bundle_format" => "split-tar-gzip-index", "archive_sha256" => "f" * 64, "archive_bytes" => 3_000_000_000, + "bundle_parts" => [{"name" => "evidence.tar.gz.part001", "sha256" => "e" * 64, "bytes" => 1_900_000_000}], "scope" => "Available logs only", "unavailable_workflow_evidence" => ["Original phase JSON was never uploaded"]} BenchmarkCases.write_json(File.join(@root, "migration/evidence.json"), {"exports" => [archive]}) result = entry From 46ae86312400a65bbb119167b88f9b7a84f08c0f Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 16:29:26 +0100 Subject: [PATCH 10/12] Retain completed PostHog and Helix qualification --- docs/cadence-qualification-2026-10-05.md | 12 +- docs/cadence.md | 4 +- .../cadence-qualification-2026-10-05.json | 4 +- migration/evidence.json | 14 + .../completions/37325075463.json | 13 + .../helix-nix/cadence-fresh-01/report.json | 761 +++++++++++++++++- results/helix-nix/cadence-fresh-01/report.md | 21 +- .../runs/2-boringcache-cold.json | 194 +++++ .../runs/2-boringcache-warm.json | 195 +++++ .../cadence-fresh-01/runs/2-cachix-cold.json | 152 ++++ .../cadence-fresh-01/runs/2-cachix-warm.json | 152 ++++ .../completions/37323164071.json | 13 + .../report.json | 509 ++++++++++++ .../cadence-rolling-advance-layers/report.md | 29 + .../runs/1-actions-cache-commit.json | 199 +++++ .../runs/1-boringcache-commit.json | 245 ++++++ 16 files changed, 2460 insertions(+), 57 deletions(-) create mode 100644 results/helix-nix/cadence-fresh-01/completions/37325075463.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json create mode 100644 results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json create mode 100644 results/posthog/cadence-rolling-advance-layers/completions/37323164071.json create mode 100644 results/posthog/cadence-rolling-advance-layers/report.json create mode 100644 results/posthog/cadence-rolling-advance-layers/report.md create mode 100644 results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json create mode 100644 results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json diff --git a/docs/cadence-qualification-2026-10-05.md b/docs/cadence-qualification-2026-10-05.md index 0a24e9bb4..3646680fd 100644 --- a/docs/cadence-qualification-2026-10-05.md +++ b/docs/cadence-qualification-2026-10-05.md @@ -1,7 +1,7 @@ # Cadence qualification — October 5, 2026 -Fifteen of seventeen rolling targets have passed a seed run and an actual source -advancement with both providers. PostHog and Zed Nix are still running. This is +Sixteen of seventeen rolling targets have passed a seed run and an actual source +advancement with both providers. Zed Nix is still running. This is an operational correctness review; these single-sample pairs do not establish comparative performance. All runs use `vcli-canary-7a5b27146ebe`. @@ -11,6 +11,12 @@ failed and cancelled cohorts remain in `results/`; corrected cohorts have new plans. The [evidence inventory](../migration/evidence.json) links archives that were downloaded again and verified after publication. +Large archives use a JSON index as `bundle_url`. Its `sha256` and `bundle_bytes` +describe that index; `archive_sha256` and `archive_bytes` describe the reconstructed +tar.gz file. Download and verify each listed part, concatenate the parts in order, +verify the resulting archive, then extract it. The hosted preservation workflow +performs those checks and verifies every exported file before recording success. + | Case / variant | Seed | Changed source | Result | | --- | --- | --- | --- | | immich | [37321739971](https://github.com/boringcache/benchmarks/actions/runs/37321739971) | [37323121773](https://github.com/boringcache/benchmarks/actions/runs/37323121773) | Passed | @@ -20,7 +26,7 @@ were downloaded again and verified after publication. | n8n / docker | [37321758035](https://github.com/boringcache/benchmarks/actions/runs/37321758035) | [37323145174](https://github.com/boringcache/benchmarks/actions/runs/37323145174) | Passed | | n8n / runners | [37321762921](https://github.com/boringcache/benchmarks/actions/runs/37321762921) | [37323150431](https://github.com/boringcache/benchmarks/actions/runs/37323150431) | Passed | | n8n / turbo | [37321767409](https://github.com/boringcache/benchmarks/actions/runs/37321767409) | [37323155106](https://github.com/boringcache/benchmarks/actions/runs/37323155106) | Passed | -| posthog / layers | [37321772493](https://github.com/boringcache/benchmarks/actions/runs/37321772493) | [37323164071](https://github.com/boringcache/benchmarks/actions/runs/37323164071) | Pending | +| posthog / layers | [37321772493](https://github.com/boringcache/benchmarks/actions/runs/37321772493) | [37323164071](https://github.com/boringcache/benchmarks/actions/runs/37323164071) | Passed | | executorch-buck2 | [37324032100](https://github.com/boringcache/benchmarks/actions/runs/37324032100) | [37324204556](https://github.com/boringcache/benchmarks/actions/runs/37324204556) | Passed | | gogs-moon | [37325135541](https://github.com/boringcache/benchmarks/actions/runs/37325135541) | [37325282767](https://github.com/boringcache/benchmarks/actions/runs/37325282767) | Passed | | opencut-moon | [37325140807](https://github.com/boringcache/benchmarks/actions/runs/37325140807) | [37325287254](https://github.com/boringcache/benchmarks/actions/runs/37325287254) | Passed | diff --git a/docs/cadence.md b/docs/cadence.md index f4e15d602..60d62200e 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -104,8 +104,8 @@ or restored. Remaining activation requirements: -- Finish PostHog and Zed Nix rolling qualification and evidence preservation. - The other fifteen rolling targets have passed seed and changed-source runs; +- Finish Zed Nix rolling qualification and evidence preservation. + The other sixteen rolling targets have passed seed and changed-source runs; hosted checks and 252 local tests pass. - Pass the corrected Zed Nix cold/warm screen using the common dependency seed. - Verify main-branch workflow-token publication at cutover. The diff --git a/migration/cadence-qualification-2026-10-05.json b/migration/cadence-qualification-2026-10-05.json index c9c846381..09ab68cd5 100644 --- a/migration/cadence-qualification-2026-10-05.json +++ b/migration/cadence-qualification-2026-10-05.json @@ -340,9 +340,9 @@ ] }, "cache_scope": "cadence-20261005-posthog-layers", - "completion": "awaiting-completion-evidence" + "completion": "success" }, - "state": "pending", + "state": "executions-passed", "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" }, { diff --git a/migration/evidence.json b/migration/evidence.json index ecbf063c9..a22972b63 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -2423,6 +2423,20 @@ "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", "verified_on": "2026-10-05", "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323164071, + "verified_files": 12, + "gaps": [], + "sha256": "ff8ac7d32632dd15f7cd666a65b959b6c4c5b542bb2160485fac9f8cf6494553", + "bundle_bytes": 283616, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323164071.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" } ] } diff --git a/results/helix-nix/cadence-fresh-01/completions/37325075463.json b/results/helix-nix/cadence-fresh-01/completions/37325075463.json new file mode 100644 index 000000000..ad890f297 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/completions/37325075463.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37325075463", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325075463", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "75038a56e31f991df749665d605a95fc42c71a9b68c90c9bd5545534c0df3a25" +} diff --git a/results/helix-nix/cadence-fresh-01/report.json b/results/helix-nix/cadence-fresh-01/report.json index 71a10214c..4b1fca7ad 100644 --- a/results/helix-nix/cadence-fresh-01/report.json +++ b/results/helix-nix/cadence-fresh-01/report.json @@ -1,8 +1,8 @@ { "schema_version": 1, "plan_sha256": "c2a0ba9332802d28a66f0938b4fd31a43ff928f6d8e7348f2f07f37e1c933421", - "complete": false, - "valid_for_comparison": false, + "complete": true, + "valid_for_comparison": true, "failures": [], "methodology_issues": [], "execution_verified": true, @@ -21,42 +21,34 @@ "status": "success", "errors": [], "evidence_manifest_sha256": "e103123da0dfd86047c1780a4ea86e37c4d55786742b3fb66d000554d9172fd0" + }, + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37325075463", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37325075463", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "75038a56e31f991df749665d605a95fc42c71a9b68c90c9bd5545534c0df3a25" } ], "publication": "unreviewed", "evidence_preservation": "requires-review", "primary_metric": "build_and_reuse_seconds", - "missing": [ - [ - 2, - "cold", - "boringcache" - ], - [ - 2, - "cold", - "cachix" - ], - [ - 2, - "warm", - "boringcache" - ], - [ - 2, - "warm", - "cachix" - ] - ], + "missing": [], "summaries": [ { "phase": "cold", "provider": "boringcache", - "count": 1, + "count": 2, "measurement": { - "median": 219.9609663066821, + "median": 281.6677286738425, "min": 219.9609663066821, - "max": 219.9609663066821 + "max": 343.3744910410028 }, "storage": null, "storage_measured_count": 0 @@ -64,27 +56,27 @@ { "phase": "warm", "provider": "boringcache", - "count": 1, + "count": 2, "measurement": { - "median": 6.481956831083494, + "median": 7.060009012536909, "min": 6.481956831083494, - "max": 6.481956831083494 + "max": 7.638061193990325 }, "storage": { - "median": 46171613, + "median": 46171613.0, "min": 46171613, "max": 46171613 }, - "storage_measured_count": 1 + "storage_measured_count": 2 }, { "phase": "cold", "provider": "cachix", - "count": 1, + "count": 2, "measurement": { - "median": 363.82386535294876, + "median": 367.90909346482397, "min": 363.82386535294876, - "max": 363.82386535294876 + "max": 371.99432157669924 }, "storage": null, "storage_measured_count": 0 @@ -92,10 +84,10 @@ { "phase": "warm", "provider": "cachix", - "count": 1, + "count": 2, "measurement": { - "median": 3.6484187521867284, - "min": 3.6484187521867284, + "median": 3.44525782967267, + "min": 3.2420969071586114, "max": 3.6484187521867284 }, "storage": null, @@ -795,6 +787,699 @@ "evidence_links": [ "https://github.com/boringcache/benchmarks/actions/runs/37320867908" ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.5731074810028076, + "build_seconds": 339.80138356, + "workflow_seconds": null, + "total_seconds": 343.3744910410028, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 343.3744910410028 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174628", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.2652809619903564, + "build_seconds": 0.37278023199996824, + "workflow_seconds": null, + "total_seconds": 7.638061193990325, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.638061193990325 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174648", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4242477416992188, + "build_seconds": 369.570073835, + "workflow_seconds": null, + "total_seconds": 371.99432157669924, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 371.99432157669924 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174627", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4607598781585693, + "build_seconds": 0.7813370290000421, + "workflow_seconds": null, + "total_seconds": 3.2420969071586114, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.2420969071586114 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174647", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] } ], "exclusions": [] diff --git a/results/helix-nix/cadence-fresh-01/report.md b/results/helix-nix/cadence-fresh-01/report.md index 67b771b52..4048f4de0 100644 --- a/results/helix-nix/cadence-fresh-01/report.md +++ b/results/helix-nix/cadence-fresh-01/report.md @@ -4,7 +4,7 @@ Question: Does Nix restore the Helix package closure on a fresh store? Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. -Observations: 4/8 recorded; 0 failed; 4 missing. +Observations: 8/8 recorded; 0 failed; 0 missing. Completion checks: passed. @@ -12,17 +12,10 @@ Publication: unreviewed. | Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | | --- | --- | ---: | ---: | --- | ---: | ---: | -| Cold build | BoringCache | 1 | 219.9609663066821 | 219.9609663066821–219.9609663066821 | unmeasured | 0 | -| Warm build | BoringCache | 1 | 6.481956831083494 | 6.481956831083494–6.481956831083494 | 46171613 | 1 | -| Cold build | Cachix | 1 | 363.82386535294876 | 363.82386535294876–363.82386535294876 | unmeasured | 0 | -| Warm build | Cachix | 1 | 3.6484187521867284 | 3.6484187521867284–3.6484187521867284 | unmeasured | 0 | - -Missing observations: - -- Sample 2, BoringCache, Cold build -- Sample 2, Cachix, Cold build -- Sample 2, BoringCache, Warm build -- Sample 2, Cachix, Warm build +| Cold build | BoringCache | 2 | 281.6677286738425 | 219.9609663066821–343.3744910410028 | unmeasured | 0 | +| Warm build | BoringCache | 2 | 7.060009012536909 | 6.481956831083494–7.638061193990325 | 46171613.0 | 2 | +| Cold build | Cachix | 2 | 367.90909346482397 | 363.82386535294876–371.99432157669924 | unmeasured | 0 | +| Warm build | Cachix | 2 | 3.44525782967267 | 3.2420969071586114–3.6484187521867284 | unmeasured | 0 | ## Observations @@ -32,5 +25,9 @@ Missing observations: | 1 | BoringCache | Warm build | 5.811720848083496 | 0.6702359829999978 | 6.481956831083494 | 46171613 | boringcache-check | hit | [JSON](runs/1-boringcache-warm.json) | | 1 | Cachix | Cold build | 2.4744367599487305 | 361.349428593 | 363.82386535294876 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | | 1 | Cachix | Warm build | 2.3420071601867676 | 1.3064115919999608 | 3.6484187521867284 | unmeasured | unmeasured | hit | [JSON](runs/1-cachix-warm.json) | +| 2 | BoringCache | Cold build | 3.5731074810028076 | 339.80138356 | 343.3744910410028 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 7.2652809619903564 | 0.37278023199996824 | 7.638061193990325 | 46171613 | boringcache-check | hit | [JSON](runs/2-boringcache-warm.json) | +| 2 | Cachix | Cold build | 2.4242477416992188 | 369.570073835 | 371.99432157669924 | unmeasured | unmeasured | miss | [JSON](runs/2-cachix-cold.json) | +| 2 | Cachix | Warm build | 2.4607598781585693 | 0.7813370290000421 | 3.2420969071586114 | unmeasured | unmeasured | hit | [JSON](runs/2-cachix-warm.json) | [Full records and checks](report.json) diff --git a/results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..88d564606 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.5731074810028076, + "build_seconds": 339.80138356, + "workflow_seconds": null, + "total_seconds": 343.3744910410028, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 343.3744910410028 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174628", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..0e0f908b8 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.2652809619903564, + "build_seconds": 0.37278023199996824, + "workflow_seconds": null, + "total_seconds": 7.638061193990325, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.638061193990325 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46171613, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "complete": true, + "measured_bytes": 46171613, + "unmeasured_tags": [], + "total_bytes": 46171613, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "tag": "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46171613, + "compressed_size": 46171613, + "size": 46171613 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-cadence-fresh-01-s2-r37325075463-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174648", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json b/results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json new file mode 100644 index 000000000..f24845015 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4242477416992188, + "build_seconds": 369.570073835, + "workflow_seconds": null, + "total_seconds": 371.99432157669924, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 371.99432157669924 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174627", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json b/results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json new file mode 100644 index 000000000..79226c16d --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/runs/2-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.4607598781585693, + "build_seconds": 0.7813370290000421, + "workflow_seconds": null, + "total_seconds": 3.2420969071586114, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 3.2420969071586114 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37325075463", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174647", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37325075463-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37325075463" + ] +} diff --git a/results/posthog/cadence-rolling-advance-layers/completions/37323164071.json b/results/posthog/cadence-rolling-advance-layers/completions/37323164071.json new file mode 100644 index 000000000..24b1ef939 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/completions/37323164071.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-advance-layers", + "sample": 1, + "run_id": "37323164071", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c6579870875c3dc988763f5253e3db3ae58ed14ff09d4f9a7663af89e3baeb54" +} diff --git a/results/posthog/cadence-rolling-advance-layers/report.json b/results/posthog/cadence-rolling-advance-layers/report.json new file mode 100644 index 000000000..8bca1180b --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/report.json @@ -0,0 +1,509 @@ +{ + "schema_version": 1, + "plan_sha256": "9c9f99841fba0c7a6826bcd1d39d5d5268e5aa213c5e838ed0bf08b4e7d23ecb", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "cadence-rolling-advance-layers", + "sample": 1, + "run_id": "37323164071", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323164071", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c6579870875c3dc988763f5253e3db3ae58ed14ff09d4f9a7663af89e3baeb54" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 2068, + "min": 2068, + "max": 2068 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 904, + "min": 904, + "max": 904 + }, + "storage": { + "median": 6262453567, + "min": 6262453567, + "max": 6262453567 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 2060, + "workflow_seconds": 2084, + "total_seconds": 2068, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2068 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174633", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 904, + "workflow_seconds": 936, + "total_seconds": 904, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 904 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6262453567, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6262453567, + "unmeasured_tags": [], + "total_bytes": 6262453567, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "37d3eb0c-7f24-4221-93c8-7cb379c43930", + "manifest_root_digest": "sha256:6fadbfd68c6194cc8d8992153cdb027cdebf4ccba7fa6b02b1bda1429e74977a", + "kv_total_size": 0, + "compressed_size": 6262453567, + "size": 6262453567 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174632", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/cadence-rolling-advance-layers/report.md b/results/posthog/cadence-rolling-advance-layers/report.md new file mode 100644 index 000000000..da070d0c9 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/report.md @@ -0,0 +1,29 @@ +# posthog: cadence-rolling-advance-layers + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | Actions Cache | 1 | 2068 | 2068–2068 | unmeasured | 0 | +| Changed-source build | BoringCache | 1 | 904 | 904–904 | 6262453567 | 1 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Changed-source build | 8 | 2060 | 2068 | unmeasured | unmeasured | not reported | [JSON](runs/1-actions-cache-commit.json) | +| 1 | BoringCache | Changed-source build | 0 | 904 | 904 | 6262453567 | boringcache-check | reuse not measured (2 refs planned) | [JSON](runs/1-boringcache-commit.json) | + +[Full records and checks](report.json) diff --git a/results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json b/results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json new file mode 100644 index 000000000..79146c225 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/runs/1-actions-cache-commit.json @@ -0,0 +1,199 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 2060, + "workflow_seconds": 2084, + "total_seconds": 2068, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 2068 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-rolling-cadence-20261005-posthog-layers", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174633", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] +} diff --git a/results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json b/results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json new file mode 100644 index 000000000..1cb68df17 --- /dev/null +++ b/results/posthog/cadence-rolling-advance-layers/runs/1-boringcache-commit.json @@ -0,0 +1,245 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "3dee2688af690830f15f0516a98dfe05bdcf0f4127db435d8e7b17c9d8684f3e", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "a6f293b70c41b00c8af78275619bab0ada2d0a4a", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 904, + "workflow_seconds": 936, + "total_seconds": 904, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 904 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "posthog-rolling-cadence-20261005-posthog-layers-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + }, + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6262453567, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6262453567, + "unmeasured_tags": [], + "total_bytes": 6262453567, + "observations": [ + { + "requested_tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "tag": "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "37d3eb0c-7f24-4221-93c8-7cb379c43930", + "manifest_root_digest": "sha256:6fadbfd68c6194cc8d8992153cdb027cdebf4ccba7fa6b02b1bda1429e74977a", + "kv_total_size": 0, + "compressed_size": 6262453567, + "size": 6262453567 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "a6f293b70c41b00c8af78275619bab0ada2d0a4a" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-rolling-cadence-20261005-posthog-layers-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323164071", + "run_attempt": "1", + "job": "publish", + "workflow": "Native comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174632", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323164071-1", + "series": { + "id": "cadence-rolling-advance-layers", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323164071" + ] +} From 2f2b023a14073000c232d2fc51e599c85d0f4242 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 16:32:49 +0100 Subject: [PATCH 11/12] Link the second verified Helix archive --- migration/evidence.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/migration/evidence.json b/migration/evidence.json index a22972b63..0822bcf5a 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -2437,6 +2437,20 @@ "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", "verified_on": "2026-10-05", "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37325075463, + "verified_files": 22, + "gaps": [], + "sha256": "d762c7f83e8e840169fbd71436ba075cd617ebf465d7fea7acf0f39a754a8163", + "bundle_bytes": 1101648302, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37325075463.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" } ] } From 2765972b2754ef72d0cc836d2988022d05bd4602 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 19:24:02 +0100 Subject: [PATCH 12/12] Complete Zed cadence qualification and preserve evidence --- docs/cadence-qualification-2026-10-05.md | 9 +- docs/cadence.md | 10 +- .../cadence-qualification-2026-10-05.json | 6 +- migration/evidence.json | 148 ++ .../completions/37320869828.json | 13 + .../completions/37327069343.json | 13 + results/zed-nix/cadence-fresh-01/report.json | 1486 +++++++++++++++++ results/zed-nix/cadence-fresh-01/report.md | 33 + .../runs/1-boringcache-cold.json | 194 +++ .../runs/1-boringcache-warm.json | 195 +++ .../cadence-fresh-01/runs/1-cachix-cold.json | 152 ++ .../cadence-fresh-01/runs/1-cachix-warm.json | 152 ++ .../runs/2-boringcache-cold.json | 194 +++ .../runs/2-boringcache-warm.json | 195 +++ .../cadence-fresh-01/runs/2-cachix-cold.json | 152 ++ .../cadence-fresh-01/runs/2-cachix-warm.json | 152 ++ .../completions/37323174641.json | 13 + .../cadence-rolling-advance/report.json | 412 +++++ .../zed-nix/cadence-rolling-advance/report.md | 29 + .../runs/1-boringcache-commit.json | 195 +++ .../runs/1-cachix-commit.json | 152 ++ .../completions/37322636399.json | 13 + .../zed-nix/cadence-rolling-seed/report.json | 407 +++++ .../zed-nix/cadence-rolling-seed/report.md | 29 + .../runs/1-boringcache-commit.json | 194 +++ .../runs/1-cachix-commit.json | 152 ++ 26 files changed, 4687 insertions(+), 13 deletions(-) create mode 100644 results/zed-nix/cadence-fresh-01/completions/37320869828.json create mode 100644 results/zed-nix/cadence-fresh-01/completions/37327069343.json create mode 100644 results/zed-nix/cadence-fresh-01/report.json create mode 100644 results/zed-nix/cadence-fresh-01/report.md create mode 100644 results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json create mode 100644 results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json create mode 100644 results/zed-nix/cadence-rolling-advance/completions/37323174641.json create mode 100644 results/zed-nix/cadence-rolling-advance/report.json create mode 100644 results/zed-nix/cadence-rolling-advance/report.md create mode 100644 results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json create mode 100644 results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json create mode 100644 results/zed-nix/cadence-rolling-seed/completions/37322636399.json create mode 100644 results/zed-nix/cadence-rolling-seed/report.json create mode 100644 results/zed-nix/cadence-rolling-seed/report.md create mode 100644 results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json create mode 100644 results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json diff --git a/docs/cadence-qualification-2026-10-05.md b/docs/cadence-qualification-2026-10-05.md index 3646680fd..77978ad42 100644 --- a/docs/cadence-qualification-2026-10-05.md +++ b/docs/cadence-qualification-2026-10-05.md @@ -1,7 +1,7 @@ # Cadence qualification — October 5, 2026 -Sixteen of seventeen rolling targets have passed a seed run and an actual source -advancement with both providers. Zed Nix is still running. This is +All seventeen rolling targets have passed a seed run and an actual source +advancement with both providers. Zed Nix also passed both fresh samples. This is an operational correctness review; these single-sample pairs do not establish comparative performance. All runs use `vcli-canary-7a5b27146ebe`. @@ -35,7 +35,7 @@ performs those checks and verifies every exported file before recording success. | obs-studio / ccache | [37323465858](https://github.com/boringcache/benchmarks/actions/runs/37323465858) | [37324219008](https://github.com/boringcache/benchmarks/actions/runs/37324219008) | Passed | | obs-studio / xcode | [37326133177](https://github.com/boringcache/benchmarks/actions/runs/37326133177) | [37326340221](https://github.com/boringcache/benchmarks/actions/runs/37326340221) | Passed | | helix-nix | [37322604229](https://github.com/boringcache/benchmarks/actions/runs/37322604229) | [37323117664](https://github.com/boringcache/benchmarks/actions/runs/37323117664) | Passed | -| zed-nix | [37322636399](https://github.com/boringcache/benchmarks/actions/runs/37322636399) | [37323174641](https://github.com/boringcache/benchmarks/actions/runs/37323174641) | Pending | +| zed-nix | [37322636399](https://github.com/boringcache/benchmarks/actions/runs/37322636399) | [37323174641](https://github.com/boringcache/benchmarks/actions/runs/37323174641) | Passed | The five REAPI comparator archives identify the intended previous run and source in `rolling-seed.json`. Both OBS comparator archives likewise identify their @@ -48,8 +48,7 @@ reported zero native hits. A further actual upstream revision in reported 70 native hits and 2 misses. Both observations remain retained. Changed source does not guarantee a hit; absence of a hit must not be reported as reuse. -Helix Nix passed two fresh cold/warm samples and its rolling pair. Zed Nix fresh -and rolling qualification remains pending. Each Nix run imports one checksummed +Helix Nix passed two fresh cold/warm samples and its rolling pair. Zed Nix passed both fresh cold/warm samples and its rolling pair. Each Nix run imports one checksummed dependency store into its provider workers and checks the dependency baseline. The measured package is excluded from that seed. A changed package derivation may require a build even when the rolling provider cache is available. diff --git a/docs/cadence.md b/docs/cadence.md index 60d62200e..5aafb45fd 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -102,12 +102,12 @@ The repository's Actions cache limit is 200 GB, with seven-day retention. Capacity alone does not establish that a particular rolling seed was retained or restored. -Remaining activation requirements: +All seventeen rolling targets passed seed and changed-source runs. Helix and +Zed Nix each passed two fresh cold/warm samples with the common dependency seed. +Sixty qualification and rehearsal archives have been published and verified; +hosted checks and 252 local tests pass. -- Finish Zed Nix rolling qualification and evidence preservation. - The other sixteen rolling targets have passed seed and changed-source runs; - hosted checks and 252 local tests pass. -- Pass the corrected Zed Nix cold/warm screen using the common dependency seed. +Remaining activation requirements: - Verify main-branch workflow-token publication at cutover. The [isolated publication rehearsal](../migration/rehearsals/mastodon-publication/review.md) passed conditional commits, dispatch and reconciliation with the operator's diff --git a/migration/cadence-qualification-2026-10-05.json b/migration/cadence-qualification-2026-10-05.json index 09ab68cd5..d59755659 100644 --- a/migration/cadence-qualification-2026-10-05.json +++ b/migration/cadence-qualification-2026-10-05.json @@ -909,7 +909,7 @@ ] }, "cache_scope": "cadence-20261005-zed-nix", - "completion": "awaiting-completion-evidence" + "completion": "success" }, "advance": { "series": "results/zed-nix/cadence-rolling-advance", @@ -927,9 +927,9 @@ ] }, "cache_scope": "cadence-20261005-zed-nix", - "completion": "awaiting-completion-evidence" + "completion": "success" }, - "state": "pending", + "state": "executions-passed", "scope": "One seed and one actual source advancement for both providers; correctness and retained-cache operation, not a comparative performance qualification" } ] diff --git a/migration/evidence.json b/migration/evidence.json index 0822bcf5a..0fab25bf3 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -2451,6 +2451,154 @@ "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", "verified_on": "2026-10-05", "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37327069343, + "verified_files": 22, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "31428770e26114ffca4544d0a1690b3788d06347bbda9deb75e2ceb4969ee04d", + "archive_bytes": 4449826654, + "archive_sha256": "6d8b1992002a2a8eddd65bf7b907a86d6a253ace6d6781d033164cc11c9eec29", + "bundle_parts": [ + { + "name": "benchmarks-37327069343.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.part001", + "bytes": 1900000000, + "sha256": "b602d6ca0db9d499dbcd01cdd468c7e2daccd0092b75edab5d3965e02d35ad4b" + }, + { + "name": "benchmarks-37327069343.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.part002", + "bytes": 1900000000, + "sha256": "95af559c1621c2d4da752dff1ee071eaf29fe78c036e154d37d7fcc7f62ea8ea" + }, + { + "name": "benchmarks-37327069343.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37327069343.tar.gz.part003", + "bytes": 649826654, + "sha256": "840dc324c77d027725f1206ddc21b088f8eca1dcf5d5a765183f1e692d150640" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37320869828, + "verified_files": 22, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "439a6c1df950844804eff84b809058f83c942adaffab05f57898fcfe2f2ca21a", + "archive_bytes": 4449816162, + "archive_sha256": "eb77ba7aa040779bf26e7c392908510da97f2c69661012f5e35e3135bb12e631", + "bundle_parts": [ + { + "name": "benchmarks-37320869828.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.part001", + "bytes": 1900000000, + "sha256": "2c2dcf9728a1046176a60b3eff3a9dcf7f8dd96a741f43e35946c28713f54639" + }, + { + "name": "benchmarks-37320869828.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.part002", + "bytes": 1900000000, + "sha256": "116e6a0a753bd348572de3b7415212dd2ba8e10d26a7bc7d3c2f11d39e3ca2fc" + }, + { + "name": "benchmarks-37320869828.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37320869828.tar.gz.part003", + "bytes": 649816162, + "sha256": "b4d202109538cdadc5854454497be2a0c25daa137c790a3509f51612723c742c" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37323174641, + "verified_files": 17, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "97f8de554fecc282a1aec272ee0144f9c1b37340b363d2608729f73c4105e83b", + "archive_bytes": 4399876552, + "archive_sha256": "48b7c9ecea8aeae7137e5993b1b69a2d68bcadc85c9e671995ca11c760991064", + "bundle_parts": [ + { + "name": "benchmarks-37323174641.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.part001", + "bytes": 1900000000, + "sha256": "0c81dcfc13451dad46a56166d05949db7c10fc6a056ad63c30e4ca041e90cdc7" + }, + { + "name": "benchmarks-37323174641.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.part002", + "bytes": 1900000000, + "sha256": "fba47b80d34745ec7dc91c2bd6ca2fa40915b2d75ca8ac72bf81e94aea99e023" + }, + { + "name": "benchmarks-37323174641.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37323174641.tar.gz.part003", + "bytes": 599876552, + "sha256": "c1e3a6d1ba77c928ac09a55fa4c59b0fcc879220344bb93a56189ce6601cfaa0" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37322636399, + "verified_files": 17, + "gaps": [], + "inventory_state": "complete", + "bundle_format": "split-tar-gzip-index", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.json", + "bundle_bytes": 1361, + "sha256": "1567a96ab8c6dacd98c285aadf34baae4b9ebe388ffcc35bdeb73a8d3e21e6da", + "archive_bytes": 4449189479, + "archive_sha256": "080b578d8fd21f65c302a24cce44f03770df87c8b3a7ecfb14e3f9f745819da9", + "bundle_parts": [ + { + "name": "benchmarks-37322636399.tar.gz.part001", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.part001", + "bytes": 1900000000, + "sha256": "883a9acbdc5b5b3187e16dc719471393ef6b2762d539d2ee9213041344ecb53d" + }, + { + "name": "benchmarks-37322636399.tar.gz.part002", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.part002", + "bytes": 1900000000, + "sha256": "4d711d9c693e459f98e9391ec923c1e53d3fff88c6b207dbf78506ba1f2a9e39" + }, + { + "name": "benchmarks-37322636399.tar.gz.part003", + "url": "https://github.com/boringcache/benchmarks/releases/download/evidence-cadence-2026-10-05/benchmarks-37322636399.tar.gz.part003", + "bytes": 649189479, + "sha256": "a3d32aa094b484d950e84530dadc17e29185c81bf11d8f8a29bc6a4f0ffcbc09" + } + ], + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-cadence-2026-10-05", + "verified_on": "2026-10-05", + "publication_verification": "Downloaded the published index and every part, verified their SHA-256 values, reconstructed and verified the archive, then verified every listed file and the run inventory", + "scope": "run, attempts, jobs, logs, artifacts, commit, workflow" } ] } diff --git a/results/zed-nix/cadence-fresh-01/completions/37320869828.json b/results/zed-nix/cadence-fresh-01/completions/37320869828.json new file mode 100644 index 000000000..bd6642e11 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/completions/37320869828.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320869828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320869828", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "62d6023079015ea03d9f488012cd18237a46f64bd5d88cef4fba3dbefd7d3033" +} diff --git a/results/zed-nix/cadence-fresh-01/completions/37327069343.json b/results/zed-nix/cadence-fresh-01/completions/37327069343.json new file mode 100644 index 000000000..a26b759ad --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/completions/37327069343.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37327069343", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37327069343", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "658d34d890ec9f6a5e8a11dfb1e8350fc40db343c73e8c4752a304d467e26f6a" +} diff --git a/results/zed-nix/cadence-fresh-01/report.json b/results/zed-nix/cadence-fresh-01/report.json new file mode 100644 index 000000000..caceb26b6 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/report.json @@ -0,0 +1,1486 @@ +{ + "schema_version": 1, + "plan_sha256": "97e97ded96f1686fba28581b92a015bace71e0fcaf38d7ee82235e290ab4f7cd", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 1, + "run_id": "37320869828", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320869828", + "source_sha": "a36ecc7823847d3b4c1bb9b480b8b157534ced91", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "62d6023079015ea03d9f488012cd18237a46f64bd5d88cef4fba3dbefd7d3033" + }, + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "sample": 2, + "run_id": "37327069343", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37327069343", + "source_sha": "a7d4c03f6e18931e07e20e6e9149c9ee30f99713", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "658d34d890ec9f6a5e8a11dfb1e8350fc40db343c73e8c4752a304d467e26f6a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 2823.3084382041816, + "min": 2792.4297531088596, + "max": 2854.1871232995036 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 9.83641568732537, + "min": 8.955559792726945, + "max": 10.717271581923796 + }, + "storage": { + "median": 576291607.0, + "min": 576291607, + "max": 576291607 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 2, + "measurement": { + "median": 2839.42401810587, + "min": 2830.319037953921, + "max": 2848.528998257819 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "cachix", + "count": 2, + "measurement": { + "median": 5.50904610590473, + "min": 4.8552112620657795, + "max": 6.1628809497436805 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.01418137550354, + "build_seconds": 2851.172941924, + "workflow_seconds": null, + "total_seconds": 2854.1871232995036, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2854.1871232995036 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174667", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 9.198845624923706, + "build_seconds": 1.5184259570000904, + "workflow_seconds": null, + "total_seconds": 10.717271581923796, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 10.717271581923796 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174707", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.9721765518188477, + "build_seconds": 2845.556821706, + "workflow_seconds": null, + "total_seconds": 2848.528998257819, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2848.528998257819 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174668", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.497649908065796, + "build_seconds": 2.3575613539999836, + "workflow_seconds": null, + "total_seconds": 4.8552112620657795, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 4.8552112620657795 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174708", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.295579671859741, + "build_seconds": 2788.134173437, + "workflow_seconds": null, + "total_seconds": 2792.4297531088596, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2792.4297531088596 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174703", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.493867635726929, + "build_seconds": 1.4616921570000159, + "workflow_seconds": null, + "total_seconds": 8.955559792726945, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 8.955559792726945 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174728", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.5098493099212646, + "build_seconds": 2827.8091886439997, + "workflow_seconds": null, + "total_seconds": 2830.319037953921, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2830.319037953921 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174702", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.8624181747436523, + "build_seconds": 3.300462775000028, + "workflow_seconds": null, + "total_seconds": 6.1628809497436805, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.1628809497436805 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174729", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/cadence-fresh-01/report.md b/results/zed-nix/cadence-fresh-01/report.md new file mode 100644 index 000000000..958fe304d --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/report.md @@ -0,0 +1,33 @@ +# zed-nix: cadence-fresh-01 + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 2 | 2823.3084382041816 | 2792.4297531088596–2854.1871232995036 | unmeasured | 0 | +| Warm build | BoringCache | 2 | 9.83641568732537 | 8.955559792726945–10.717271581923796 | 576291607.0 | 2 | +| Cold build | Cachix | 2 | 2839.42401810587 | 2830.319037953921–2848.528998257819 | unmeasured | 0 | +| Warm build | Cachix | 2 | 5.50904610590473 | 4.8552112620657795–6.1628809497436805 | unmeasured | 0 | + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 3.01418137550354 | 2851.172941924 | 2854.1871232995036 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 9.198845624923706 | 1.5184259570000904 | 10.717271581923796 | 576291607 | boringcache-check | hit | [JSON](runs/1-boringcache-warm.json) | +| 1 | Cachix | Cold build | 2.9721765518188477 | 2845.556821706 | 2848.528998257819 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | +| 1 | Cachix | Warm build | 2.497649908065796 | 2.3575613539999836 | 4.8552112620657795 | unmeasured | unmeasured | hit | [JSON](runs/1-cachix-warm.json) | +| 2 | BoringCache | Cold build | 4.295579671859741 | 2788.134173437 | 2792.4297531088596 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 7.493867635726929 | 1.4616921570000159 | 8.955559792726945 | 576291607 | boringcache-check | hit | [JSON](runs/2-boringcache-warm.json) | +| 2 | Cachix | Cold build | 2.5098493099212646 | 2827.8091886439997 | 2830.319037953921 | unmeasured | unmeasured | miss | [JSON](runs/2-cachix-cold.json) | +| 2 | Cachix | Warm build | 2.8624181747436523 | 3.300462775000028 | 6.1628809497436805 | unmeasured | unmeasured | hit | [JSON](runs/2-cachix-warm.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..84fdec209 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.01418137550354, + "build_seconds": 2851.172941924, + "workflow_seconds": null, + "total_seconds": 2854.1871232995036, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2854.1871232995036 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174667", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..b56e4bb0e --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 9.198845624923706, + "build_seconds": 1.5184259570000904, + "workflow_seconds": null, + "total_seconds": 10.717271581923796, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 10.717271581923796 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s1-r37320869828-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174707", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json b/results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json new file mode 100644 index 000000000..162c691dc --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.9721765518188477, + "build_seconds": 2845.556821706, + "workflow_seconds": null, + "total_seconds": 2848.528998257819, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2848.528998257819 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174668", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json b/results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json new file mode 100644 index 000000000..7fe7cf109 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/1-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.497649908065796, + "build_seconds": 2.3575613539999836, + "workflow_seconds": null, + "total_seconds": 4.8552112620657795, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 4.8552112620657795 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37320869828", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "shared-benchmark-cadence", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174708", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37320869828-1", + "series": { + "id": "cadence-fresh-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37320869828" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..82e181f4d --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-cold.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4.295579671859741, + "build_seconds": 2788.134173437, + "workflow_seconds": null, + "total_seconds": 2792.4297531088596, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2792.4297531088596 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174703", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..6f98a4bcc --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-boringcache-warm.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.493867635726929, + "build_seconds": 1.4616921570000159, + "workflow_seconds": null, + "total_seconds": 8.955559792726945, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 8.955559792726945 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "tag": "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-cadence-fresh-01-s2-r37327069343-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174728", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json b/results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json new file mode 100644 index 000000000..db58f6946 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-cachix-cold.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.5098493099212646, + "build_seconds": 2827.8091886439997, + "workflow_seconds": null, + "total_seconds": 2830.319037953921, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2830.319037953921 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174702", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json b/results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json new file mode 100644 index 000000000..6a4bc3ee9 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/runs/2-cachix-warm.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.8624181747436523, + "build_seconds": 3.300462775000028, + "workflow_seconds": null, + "total_seconds": 6.1628809497436805, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 6.1628809497436805 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37327069343", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "cadence-fresh-qualification", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174729", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37327069343-1", + "series": { + "id": "cadence-fresh-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37327069343" + ] +} diff --git a/results/zed-nix/cadence-rolling-advance/completions/37323174641.json b/results/zed-nix/cadence-rolling-advance/completions/37323174641.json new file mode 100644 index 000000000..1f5003d42 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/completions/37323174641.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323174641", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d08da02269663b3a462ecefe0217d0b46a159fd754cd916534bdb08ed54b3f29" +} diff --git a/results/zed-nix/cadence-rolling-advance/report.json b/results/zed-nix/cadence-rolling-advance/report.json new file mode 100644 index 000000000..087da9b32 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/report.json @@ -0,0 +1,412 @@ +{ + "schema_version": 1, + "plan_sha256": "5a98dac1491c4ace612df8a0bfcf5e2c14c0f6c1697f21dd0ddb3c65fcbebef1", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-advance", + "sample": 1, + "run_id": "37323174641", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37323174641", + "source_sha": "915ea2921d4b4d23f0238a4b7e28ecce1bbcc880", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d08da02269663b3a462ecefe0217d0b46a159fd754cd916534bdb08ed54b3f29" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 1872.8299867073292, + "min": 1872.8299867073292, + "max": 1872.8299867073292 + }, + "storage": { + "median": 576291607, + "min": 576291607, + "max": 576291607 + }, + "storage_measured_count": 1 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 1802.7406344062056, + "min": 1802.7406344062056, + "max": 1802.7406344062056 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.843063831329346, + "build_seconds": 1864.9869228759999, + "workflow_seconds": null, + "total_seconds": 1872.8299867073292, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1872.8299867073292 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174735", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1445672512054443, + "build_seconds": 1800.5960671550001, + "workflow_seconds": null, + "total_seconds": 1802.7406344062056, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1802.7406344062056 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174734", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/cadence-rolling-advance/report.md b/results/zed-nix/cadence-rolling-advance/report.md new file mode 100644 index 000000000..919d375d2 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/report.md @@ -0,0 +1,29 @@ +# zed-nix: cadence-rolling-advance + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 1872.8299867073292 | 1872.8299867073292–1872.8299867073292 | 576291607 | 1 | +| Changed-source build | Cachix | 1 | 1802.7406344062056 | 1802.7406344062056–1802.7406344062056 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 7.843063831329346 | 1864.9869228759999 | 1872.8299867073292 | 576291607 | boringcache-check | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.1445672512054443 | 1800.5960671550001 | 1802.7406344062056 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json b/results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json new file mode 100644 index 000000000..e08151a2a --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/runs/1-boringcache-commit.json @@ -0,0 +1,195 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7.843063831329346, + "build_seconds": 1864.9869228759999, + "workflow_seconds": null, + "total_seconds": 1872.8299867073292, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1872.8299867073292 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": 576291607, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": true, + "measured_bytes": 576291607, + "unmeasured_tags": [], + "total_bytes": 576291607, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 576291607, + "compressed_size": 576291607, + "size": 576291607 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174735", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] +} diff --git a/results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json b/results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json new file mode 100644 index 000000000..31c3c6f26 --- /dev/null +++ b/results/zed-nix/cadence-rolling-advance/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b74d3b7ff190fc242258f2ee4b425b0661bbe915c899a6a9347dee796b2a2ecb", + "source": { + "repository": "zed-industries/zed", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "a6169ca96987105be7d60a5051f3155eb6225ccc" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1445672512054443, + "build_seconds": 1800.5960671550001, + "workflow_seconds": null, + "total_seconds": 1802.7406344062056, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 1802.7406344062056 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "a6169ca96987105be7d60a5051f3155eb6225ccc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37323174641", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174734", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37323174641-1", + "series": { + "id": "cadence-rolling-advance", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37323174641" + ] +} diff --git a/results/zed-nix/cadence-rolling-seed/completions/37322636399.json b/results/zed-nix/cadence-rolling-seed/completions/37322636399.json new file mode 100644 index 000000000..c5deac6e2 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/completions/37322636399.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322636399", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "92a34678c864473869b9cad390bfbed9cc47760dc727a0ed17f43691a744855e" +} diff --git a/results/zed-nix/cadence-rolling-seed/report.json b/results/zed-nix/cadence-rolling-seed/report.json new file mode 100644 index 000000000..1e04db5d0 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/report.json @@ -0,0 +1,407 @@ +{ + "schema_version": 1, + "plan_sha256": "bf6c8078a29447c37ad71c86489ed872363058a299dcf47768b2c5e99b7c03af", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling seed and changed-source sequence are unverified." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-rolling-seed", + "sample": 1, + "run_id": "37322636399", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37322636399", + "source_sha": "694acf2b29d055602938d869ef1815c42de6ee11", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "92a34678c864473869b9cad390bfbed9cc47760dc727a0ed17f43691a744855e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 2834.6952709450306, + "min": 2834.6952709450306, + "max": 2834.6952709450306 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "commit", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 7.971168194222969, + "min": 7.971168194222969, + "max": 7.971168194222969 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.5863049030303955, + "build_seconds": 2829.1089660420002, + "workflow_seconds": null, + "total_seconds": 2834.6952709450306, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2834.6952709450306 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174670", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1795594692230225, + "build_seconds": 5.791608724999946, + "workflow_seconds": null, + "total_seconds": 7.971168194222969, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.971168194222969 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174671", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/cadence-rolling-seed/report.md b/results/zed-nix/cadence-rolling-seed/report.md new file mode 100644 index 000000000..111104937 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/report.md @@ -0,0 +1,29 @@ +# zed-nix: cadence-rolling-seed + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed. + +Observations: 2/2 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Changed-source build | BoringCache | 1 | 2834.6952709450306 | 2834.6952709450306–2834.6952709450306 | unmeasured | 0 | +| Changed-source build | Cachix | 1 | 7.971168194222969 | 7.971168194222969–7.971168194222969 | unmeasured | 0 | + +Comparison checks: + +- Rolling seed and changed-source sequence are unverified. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Changed-source build | 5.5863049030303955 | 2829.1089660420002 | 2834.6952709450306 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-commit.json) | +| 1 | Cachix | Changed-source build | 2.1795594692230225 | 5.791608724999946 | 7.971168194222969 | unmeasured | unmeasured | not reported | [JSON](runs/1-cachix-commit.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json b/results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json new file mode 100644 index 000000000..f90bfdbf2 --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/runs/1-boringcache-commit.json @@ -0,0 +1,194 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.5863049030303955, + "build_seconds": 2829.1089660420002, + "workflow_seconds": null, + "total_seconds": 2834.6952709450306, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 2834.6952709450306 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "tag": "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-rolling-cadence-20261005-zed-nix" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174670", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] +} diff --git a/results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json b/results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json new file mode 100644 index 000000000..cd7de979e --- /dev/null +++ b/results/zed-nix/cadence-rolling-seed/runs/1-cachix-commit.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "rolling", + "phase": "commit", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e141d3511b4d0c230238f88e5ccad99954955691077867077e444c959dc823ff", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "execution": { + "sync": "upstream-head", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.1795594692230225, + "build_seconds": 5.791608724999946, + "workflow_seconds": null, + "total_seconds": 7.971168194222969, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "build_and_reuse_seconds": 7.971168194222969 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37322636399", + "run_attempt": "1", + "job": "commit", + "workflow": "Nix comparison rolling", + "ref_name": "cadence-rehearsal-rolling", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174671", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37322636399-1", + "series": { + "id": "cadence-rolling-seed", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37322636399" + ] +}