From 6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 02:45:29 +0100 Subject: [PATCH 01/17] Select workload variants through the shared benchmark workflow --- .github/actions/prepare-case/action.yml | 7 ++ .github/workflows/native-fresh-benchmark.yml | 30 +++++++- .../workflows/native-rolling-benchmark.yml | 27 ++++++- bin/bench | 2 +- cases/immich/case.json | 18 ++++- .../immich-docker-benchmark/action.yml | 14 +++- cases/mastodon/case.json | 54 ++++++++++++- .../mastodon-docker-benchmark/action.yml | 14 +++- cases/n8n/case.json | 72 +++++++++++++++++- .../actions/n8n-docker-benchmark/action.yml | 16 ++-- .../actions/n8n-turbo-benchmark/action.yml | 25 ++++-- cases/posthog/case.json | 47 +++++++++++- .../posthog-docker-benchmark/action.yml | 13 +++- docs/migration.md | 11 +++ docs/process.md | 18 +++++ .../shared-turbo-qualification-01/series.json | 45 +++++++++++ schemas/case.schema.json | 25 +++++- scripts/benchmark-cases.rb | 30 ++++++-- scripts/check-report-contract.rb | 5 +- scripts/native-case.rb | 64 ++++++++++++++-- test/native_case_test.rb | 76 +++++++++++++++++++ 21 files changed, 552 insertions(+), 61 deletions(-) create mode 100644 results/n8n/shared-turbo-qualification-01/series.json diff --git a/.github/actions/prepare-case/action.yml b/.github/actions/prepare-case/action.yml index 81ca36c54..d28e8bf6c 100644 --- a/.github/actions/prepare-case/action.yml +++ b/.github/actions/prepare-case/action.yml @@ -10,6 +10,9 @@ inputs: suffix: description: Optional diagnostic result suffix. default: '' + variant: + description: Reviewed native workload variant. + default: '' runs: using: composite steps: @@ -27,10 +30,14 @@ runs: BENCHMARK_CASE_ID: ${{ inputs.case-id }} NATIVE_LANE: ${{ inputs.native-lane }} BENCHMARK_ID_SUFFIX: ${{ inputs.suffix }} + BENCHMARK_VARIANT: ${{ inputs.variant }} BUNDLE_GEMFILE: ${{ github.workspace }}/.harness/Gemfile run: | args=() if [[ -n "$NATIVE_LANE" ]]; then args+=(--native-lane "$NATIVE_LANE" --suffix "$BENCHMARK_ID_SUFFIX") + if [[ -n "$BENCHMARK_VARIANT" ]]; then + args+=(--variant "$BENCHMARK_VARIANT") + fi fi ruby .harness/bin/bench prepare "$BENCHMARK_CASE_ID" --directory "$GITHUB_WORKSPACE" "${args[@]}" diff --git a/.github/workflows/native-fresh-benchmark.yml b/.github/workflows/native-fresh-benchmark.yml index 1ca1d26a2..6b8703f83 100644 --- a/.github/workflows/native-fresh-benchmark.yml +++ b/.github/workflows/native-fresh-benchmark.yml @@ -5,6 +5,7 @@ run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs permissions: contents: read actions: read + packages: read env: BORINGCACHE_NO_GIT: 'true' BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" @@ -36,6 +37,7 @@ jobs: case-id: "${{ inputs.case_id }}" native-lane: fresh suffix: "${{ inputs.benchmark_id_suffix }}" + variant: "${{ inputs.variant }}" - uses: "./.github/actions/benchmark-phase" with: strategy: "${{ matrix.strategy }}" @@ -45,10 +47,12 @@ jobs: cli_version: "${{ inputs.cli_version }}" buildkit_image: "${{ inputs.buildkit_image }}" cli_platform: "${{ inputs.cli_platform }}" + report_variant: "${{ inputs.variant }}" permissions: contents: read actions: read id-token: write + packages: read env: BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' && inputs.runner_label || 'ubuntu-latest' }}" @@ -78,6 +82,7 @@ jobs: case-id: "${{ inputs.case_id }}" native-lane: fresh suffix: "${{ inputs.benchmark_id_suffix }}" + variant: "${{ inputs.variant }}" - uses: "./.github/actions/benchmark-phase" with: strategy: "${{ matrix.strategy }}" @@ -87,10 +92,12 @@ jobs: cli_version: "${{ inputs.cli_version }}" buildkit_image: "${{ inputs.buildkit_image }}" cli_platform: "${{ inputs.cli_platform }}" + report_variant: "${{ inputs.variant }}" permissions: contents: read actions: read id-token: write + packages: read env: BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' && inputs.runner_label || 'ubuntu-latest' }}" @@ -111,6 +118,7 @@ jobs: case-id: "${{ inputs.case_id }}" native-lane: fresh suffix: "${{ inputs.benchmark_id_suffix }}" + variant: "${{ inputs.variant }}" - name: Download the benchmark phase evidence uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 with: @@ -125,14 +133,18 @@ jobs: - name: Retain the BoringCache fresh result uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: benchmark-${{ env.BENCHMARK_ID }}-boringcache-fresh - path: benchmark-results/${{ env.BENCHMARK_ID }}-boringcache-fresh.json + name: benchmark-${{ env.BENCHMARK_ID }}-boringcache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-fresh + path: benchmark-results/${{ env.BENCHMARK_ID }}-boringcache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-fresh.json if-no-files-found: error - name: Retain the GitHub Actions fresh result uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: benchmark-${{ env.BENCHMARK_ID }}-actions-cache-fresh - path: benchmark-results/${{ env.BENCHMARK_ID }}-actions-cache-fresh.json + name: benchmark-${{ env.BENCHMARK_ID }}-actions-cache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-fresh + path: benchmark-results/${{ env.BENCHMARK_ID }}-actions-cache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-fresh.json if-no-files-found: error env: BENCHMARK_RUNNER_CLASS: ubuntu-latest @@ -178,6 +190,12 @@ jobs: required: false type: string default: '' + variant: + type: string + required: false + default: '' + description: Reviewed native workload variant; required when the case declares + variants. workflow_call: inputs: cli_version: @@ -211,6 +229,10 @@ jobs: required: false type: string default: '' + variant: + type: string + required: false + default: '' concurrency: group: benchmark-provider-comparisons cancel-in-progress: false diff --git a/.github/workflows/native-rolling-benchmark.yml b/.github/workflows/native-rolling-benchmark.yml index 7bf26ddff..51df2f8e4 100644 --- a/.github/workflows/native-rolling-benchmark.yml +++ b/.github/workflows/native-rolling-benchmark.yml @@ -5,6 +5,7 @@ run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inpu permissions: contents: read actions: read + packages: read env: BORINGCACHE_NO_GIT: 'true' BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" @@ -36,6 +37,7 @@ jobs: case-id: "${{ inputs.case_id }}" native-lane: rolling suffix: "${{ inputs.benchmark_id_suffix }}" + variant: "${{ inputs.variant }}" - uses: "./.github/actions/benchmark-phase" with: strategy: "${{ matrix.strategy }}" @@ -45,10 +47,12 @@ jobs: cli_version: "${{ inputs.cli_version }}" buildkit_image: "${{ inputs.buildkit_image }}" cli_platform: "${{ inputs.cli_platform }}" + report_variant: "${{ inputs.variant }}" permissions: contents: read actions: read id-token: write + packages: read env: BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' && inputs.runner_label || 'ubuntu-latest' }}" @@ -69,6 +73,7 @@ jobs: case-id: "${{ inputs.case_id }}" native-lane: rolling suffix: "${{ inputs.benchmark_id_suffix }}" + variant: "${{ inputs.variant }}" - name: Download the benchmark phase evidence uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 with: @@ -83,14 +88,18 @@ jobs: - name: Retain the BoringCache rolling result uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: benchmark-${{ env.BENCHMARK_ID }}-boringcache-rolling - path: benchmark-results/${{ env.BENCHMARK_ID }}-boringcache-rolling.json + name: benchmark-${{ env.BENCHMARK_ID }}-boringcache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-rolling + path: benchmark-results/${{ env.BENCHMARK_ID }}-boringcache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-rolling.json if-no-files-found: error - name: Retain the GitHub Actions rolling result uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: benchmark-${{ env.BENCHMARK_ID }}-actions-cache-rolling - path: benchmark-results/${{ env.BENCHMARK_ID }}-actions-cache-rolling.json + name: benchmark-${{ env.BENCHMARK_ID }}-actions-cache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-rolling + path: benchmark-results/${{ env.BENCHMARK_ID }}-actions-cache${{ env.BENCHMARK_VARIANT_SUFFIX + }}-rolling.json if-no-files-found: error env: BENCHMARK_RUNNER_CLASS: ubuntu-latest @@ -136,6 +145,12 @@ jobs: required: false type: string default: '' + variant: + type: string + required: false + default: '' + description: Reviewed native workload variant; required when the case declares + variants. workflow_call: inputs: cli_version: @@ -169,6 +184,10 @@ jobs: required: false type: string default: '' + variant: + type: string + required: false + default: '' concurrency: group: benchmark-provider-comparisons cancel-in-progress: false diff --git a/bin/bench b/bin/bench index 17d9de981..fadab9940 100755 --- a/bin/bench +++ b/bin/bench @@ -83,7 +83,7 @@ begin puts JSON.pretty_generate(BenchmarkCases.dispatch(item, **options.slice(:lane, :workflow, :inputs, :ref), output: output)) when "prepare" raise BenchmarkCases::Error, "Select a case ID and --directory" unless item && options[:directory] - puts BenchmarkCases.prepare(item, **options.slice(:directory, :native_lane, :suffix)) + puts BenchmarkCases.prepare(item, **options.slice(:directory, :native_lane, :suffix, :variant)) when "contracts" raise BenchmarkCases::Error, "Use --directory for disposable contract views" unless options[:directory] BenchmarkCases.validate diff --git a/cases/immich/case.json b/cases/immich/case.json index fe9d11170..26a7bde52 100644 --- a/cases/immich/case.json +++ b/cases/immich/case.json @@ -21,14 +21,22 @@ }, "execution": { "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, { "path": ".github/workflows/immich-immich-benchmark.yml", - "lane": "rolling", + "lane": "diagnostic", "inputs": {} }, { "path": ".github/workflows/immich-immich-fresh-benchmark.yml", - "lane": "fresh", + "lane": "diagnostic", "inputs": {} } ], @@ -37,6 +45,12 @@ "cache_tags": { "immich-docker-local": "{scope}-docker", "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": {}, + "rolling_inputs": {} } }, "reporting": { diff --git a/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml b/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml index 643b26ebd..830396828 100644 --- a/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml +++ b/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Immich Docker benchmark phase description: Runs one Immich Docker cache provider and phase. inputs: @@ -18,6 +19,10 @@ inputs: buildkit_image: required: false default: '' + report_variant: + required: false + default: '' + description: Declared series workload variant. runs: using: composite steps: @@ -91,7 +96,7 @@ runs: run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Set up Buildx for the GitHub Actions comparison if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker + uses: "./.github/actions/setup-docker" - name: Start the build timer id: build_timer shell: bash @@ -125,7 +130,7 @@ runs: - name: Publish with BoringCache id: publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" with: @@ -135,7 +140,7 @@ runs: - name: Restore with BoringCache id: restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" with: @@ -168,6 +173,7 @@ runs: GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" + REPORT_VARIANT: "${{ inputs.report_variant }}" run: | set -euo pipefail ruby ./scripts/benchmark-report.rb phase \ @@ -175,7 +181,7 @@ runs: --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ --phase "$REPORT_PHASE" \ - --mode docker \ + --variant "$REPORT_VARIANT" --mode docker \ --restore-or-setup-seconds "$SETUP_SECONDS" \ --workflow-seconds "$(( $(date +%s) - WORKFLOW_STARTED_AT ))" \ --build-seconds "$BUILD_SECONDS" \ diff --git a/cases/mastodon/case.json b/cases/mastodon/case.json index 4078e1a75..a11ac481d 100644 --- a/cases/mastodon/case.json +++ b/cases/mastodon/case.json @@ -21,14 +21,28 @@ }, "execution": { "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, { "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", - "lane": "rolling", + "lane": "diagnostic", "inputs": {} }, { "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", - "lane": "fresh", + "lane": "diagnostic", "inputs": {} } ], @@ -37,6 +51,42 @@ "cache_tags": { "mastodon-docker-local": "{scope}-docker", "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } } }, "reporting": { diff --git a/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml b/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml index 3a0e01a70..1ed45d00b 100644 --- a/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml +++ b/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Mastodon Docker benchmark phase description: Runs one Mastodon Docker cache provider and phase. inputs: @@ -26,6 +27,10 @@ inputs: upstream_ref: required: false default: '' + report_variant: + required: false + default: '' + description: Declared series workload variant. runs: using: composite steps: @@ -109,7 +114,7 @@ runs: run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Set up Buildx for the GitHub Actions comparison if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker + uses: "./.github/actions/setup-docker" - name: Start the build timer id: build_timer shell: bash @@ -145,7 +150,7 @@ runs: - name: Publish with BoringCache id: publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" with: @@ -155,7 +160,7 @@ runs: - name: Restore with BoringCache id: restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" with: @@ -188,6 +193,7 @@ runs: GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" + REPORT_VARIANT: "${{ inputs.report_variant }}" run: | set -euo pipefail ruby ./scripts/benchmark-report.rb phase \ @@ -195,7 +201,7 @@ runs: --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ --phase "$REPORT_PHASE" \ - --mode docker \ + --variant "$REPORT_VARIANT" --mode docker \ --restore-or-setup-seconds "$SETUP_SECONDS" \ --workflow-seconds "$(( $(date +%s) - WORKFLOW_STARTED_AT ))" \ --build-seconds "$BUILD_SECONDS" \ diff --git a/cases/n8n/case.json b/cases/n8n/case.json index 9667adca6..0ed6fad54 100644 --- a/cases/n8n/case.json +++ b/cases/n8n/case.json @@ -21,24 +21,39 @@ }, "execution": { "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, { "path": ".github/workflows/n8n-n8n-benchmark.yml", - "lane": "rolling", + "lane": "diagnostic", "inputs": {} }, { "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", - "lane": "rolling", + "lane": "diagnostic", "inputs": {} }, { "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", - "lane": "fresh", + "lane": "diagnostic", "inputs": {} }, { "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", - "lane": "fresh", + "lane": "diagnostic", "inputs": {} } ], @@ -47,6 +62,55 @@ "cache_tags": { "n8n-docker": "{scope}-docker", "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } } }, "reporting": { diff --git a/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml b/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml index 311569a42..8f89f1762 100644 --- a/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml +++ b/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run n8n Docker benchmark phase description: Runs one n8n Docker cache provider, image case, and phase. inputs: @@ -24,6 +25,10 @@ inputs: required: true node_version: required: true + report_variant: + required: false + default: '' + description: Declared series workload variant. runs: using: composite steps: @@ -99,7 +104,7 @@ runs: run: ruby ./scripts/activate-docker-plan.rb --dockerfile "$DOCKERFILE" --node-version "$NODE_VERSION" --push "$PUSH_IMAGE" --image "$IMAGE" - name: Set up Node.js - uses: ./.github/actions/setup-node + uses: "./.github/actions/setup-node" with: node-version: "${{ inputs.node_version }}" - name: Enable the pinned pnpm release @@ -138,7 +143,7 @@ runs: run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Set up Buildx for the GitHub Actions comparison if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker + uses: "./.github/actions/setup-docker" - name: Start the build timer id: build_timer shell: bash @@ -172,7 +177,7 @@ runs: - name: Publish with BoringCache id: publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" with: @@ -183,7 +188,7 @@ runs: - name: Restore with BoringCache id: restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" with: @@ -217,6 +222,7 @@ runs: GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" + REPORT_VARIANT: "${{ inputs.report_variant }}" run: | set -euo pipefail ruby ./scripts/benchmark-report.rb phase \ @@ -224,7 +230,7 @@ runs: --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ --phase "$REPORT_PHASE" \ - --mode docker \ + --variant "$REPORT_VARIANT" --mode docker \ --restore-or-setup-seconds "$SETUP_SECONDS" \ --workflow-seconds "$(( $(date +%s) - WORKFLOW_STARTED_AT ))" \ --build-seconds "$BUILD_SECONDS" \ diff --git a/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml b/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml index 20bdd19b3..2328ce636 100644 --- a/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml +++ b/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run n8n Turbo benchmark phase description: Runs one n8n Turbo cache provider and phase. inputs: @@ -14,6 +15,10 @@ inputs: default: '' node_version: required: true + report_variant: + required: false + default: '' + description: Declared series workload variant. runs: using: composite steps: @@ -79,7 +84,7 @@ runs: echo "source_repository=n8n-io/n8n" >> "$GITHUB_OUTPUT" echo "source_sha=${source_sha}" >> "$GITHUB_OUTPUT" - name: Set up Node.js - uses: ./.github/actions/setup-node + uses: "./.github/actions/setup-node" with: node-version: "${{ inputs.node_version }}" - name: Enable the pinned pnpm release @@ -113,7 +118,7 @@ runs: - name: Publish with BoringCache id: boringcache-publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: turbo @@ -122,15 +127,15 @@ runs: - name: Restore with BoringCache id: boringcache-restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: turbo trust-policy: restore working-directory: upstream fail-on-cache-miss: true - - name: Start the build timer - id: build_timer + - name: Capture cache setup timing + id: setup_timing shell: bash env: SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" @@ -138,7 +143,6 @@ runs: set -euo pipefail now="$(date +%s)" echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - name: Install n8n dependencies shell: bash working-directory: upstream @@ -146,6 +150,10 @@ runs: CI: '1' GITHUB_TOKEN: "${{ github.token }}" run: corepack pnpm install --frozen-lockfile + - name: Start the build timer + id: build_timer + shell: bash + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Build n8n shell: bash env: @@ -183,7 +191,7 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" + SETUP_SECONDS: "${{ steps.setup_timing.outputs.setup_seconds }}" BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" CACHE_HIT: "${{ steps.boringcache-publish.outputs.cache-hit || steps.boringcache-restore.outputs.cache-hit || steps.actions_cache_0.outputs.cache-hit || steps.actions_cache_1.outputs.cache-hit @@ -194,6 +202,7 @@ runs: GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" + REPORT_VARIANT: "${{ inputs.report_variant }}" run: | set -euo pipefail ruby ./scripts/benchmark-report.rb phase --verified-output \ @@ -201,7 +210,7 @@ runs: --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ --phase "$REPORT_PHASE" \ - --mode turbo \ + --variant "$REPORT_VARIANT" --mode turbo \ --restore-or-setup-seconds "$SETUP_SECONDS" \ --workflow-seconds "$(( $(date +%s) - WORKFLOW_STARTED_AT ))" \ --build-seconds "$BUILD_SECONDS" \ diff --git a/cases/posthog/case.json b/cases/posthog/case.json index 812a3cf38..003bf7c17 100644 --- a/cases/posthog/case.json +++ b/cases/posthog/case.json @@ -21,14 +21,27 @@ }, "execution": { "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, { "path": ".github/workflows/posthog-posthog-benchmark.yml", - "lane": "rolling", + "lane": "diagnostic", "inputs": {} }, { "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", - "lane": "fresh", + "lane": "diagnostic", "inputs": {} } ], @@ -37,6 +50,36 @@ "cache_tags": { "posthog-docker-local": "{scope}-docker", "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } } }, "reporting": { diff --git a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml index 70a7f2926..922e1e238 100644 --- a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml +++ b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run PostHog Docker benchmark phase description: Runs one PostHog cache provider and phase through the public product path. @@ -67,6 +68,10 @@ inputs: description: Exact published cohort used by a warm phase. required: false default: '' + report_variant: + required: false + default: '' + description: Declared series workload variant. outputs: cache_scope: description: Published cache cohort. @@ -194,7 +199,7 @@ runs: run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Set up Buildx for the Actions cache protocol if: inputs.strategy == 'actions-cache' || inputs.strategy == 'runs-on-cache' - uses: ./.github/actions/setup-docker + uses: "./.github/actions/setup-docker" - name: Start the build timer id: build_timer shell: bash @@ -228,7 +233,7 @@ runs: id: publish if: inputs.strategy == 'boringcache' && (inputs.phase == 'publish' || (inputs.phase == 'warm' && inputs.publish_on_warm == 'true')) - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" POSTHOG_SOURCEMAP_API_KEY: "${{ inputs.posthog_sourcemap_api_key }}" @@ -241,7 +246,7 @@ runs: id: restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' && inputs.publish_on_warm != 'true' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" POSTHOG_SOURCEMAP_API_KEY: "${{ inputs.posthog_sourcemap_api_key }}" @@ -283,7 +288,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" REPORT_PHASE: "${{ inputs.report_phase || inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" - REPORT_VARIANT: "${{ steps.scope.outputs.variant }}" + REPORT_VARIANT: "${{ inputs.report_variant || steps.scope.outputs.variant }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" diff --git a/docs/migration.md b/docs/migration.md index 1e65763cf..23d42d353 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -32,6 +32,17 @@ and reporting qualification remain required. Imported execution workflows accept explicit dispatch only until their live qualification passes. Schedules and source-triggered execution must be enabled as part of the reviewed caller cutover. PR validation runs the harness guards. + +The shared native fresh path also selects individual n8n Turbo/Docker, Mastodon +server/streaming, and PostHog cache profiles from reviewed variant metadata; +Immich's server uses the same path without a variant. Their imported matrices +remain registered diagnostics while output checks and live execution are +qualified. The new Docker paths do not establish a loaded or published image, +or qualify the original rolling publication behavior. Existing callers and +schedules still use the original repositories. +n8n Turbo's primary measurement now captures cache setup and the build separately +from its dependency installation. The install still runs before the build. Earlier +results retain their original timed scope and must not be mixed with this series. GitHub's default CodeQL configuration now scans Ruby and Actions. Python was removed from its language list because this branch removes the maintained Python reporter; query suite, threat model, weekly schedule, and runner settings were diff --git a/docs/process.md b/docs/process.md index 553c6e371..65688a18d 100644 --- a/docs/process.md +++ b/docs/process.md @@ -43,6 +43,24 @@ do not copy workflow setup or reporting into each new case. Declare tag mappings in `execution.cache_tags` and use `scope-case-cache.rb`. Use `prepare-source.rb` for an unchanged pinned upstream checkout. +When one case has several workloads, declare their recipe overrides in +`execution.native.variants` and register the same variants on the shared workflow +entry with `variant_input: variant`. Select one variant before execution: + +```sh +bin/bench plan n8n --variant turbo +bin/bench prepare n8n --native-lane fresh --variant turbo --directory /tmp/n8n-turbo +bin/bench start n8n --series screening-turbo-01 --variant turbo --samples 1 +``` + +The selected variant fixes the case action, benchmark ID, toolchain inputs, and +report identity. Provider-specific boolean flags belong in `provider_flags`; +for example, the Actions Cache arm keeps BoringCache's tool-proxy flags disabled. +All variants use the same provider lifecycle and canonical reporter. Missing or +unknown variants fail before dispatch. Contract checks prepare every registered +native variant, including its report filenames. A fresh variant passing does not +qualify another variant or the original rolling matrix. + Keep different provider sets, architectures, source-change sequences, and output behavior explicit in their adapters or case actions. OBS, Cargo product proofs, and the Docker corpus retain their distinct execution shapes. Add a workflow diff --git a/results/n8n/shared-turbo-qualification-01/series.json b/results/n8n/shared-turbo-qualification-01/series.json new file mode 100644 index 000000000..de3bb8b60 --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", + "created_at": "2026-10-03T01:45:05Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "variant": "turbo", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "turbo" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "plan_sha256": "035df772551717a7a2d369d39994c744f270815a5481833e7a494a06097dcae9" +} diff --git a/schemas/case.schema.json b/schemas/case.schema.json index d72ae6291..cb07e5bde 100644 --- a/schemas/case.schema.json +++ b/schemas/case.schema.json @@ -72,7 +72,23 @@ "benchmark_id": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}, "fresh_inputs": {"type": "object", "additionalProperties": {"type": "string"}}, "rolling_inputs": {"type": "object", "additionalProperties": {"type": "string"}}, - "environment": {"type": "object", "additionalProperties": {"type": "string"}} + "environment": {"type": "object", "additionalProperties": {"type": "string"}}, + "provider_flags": {"$ref": "#/$defs/provider_flags"}, + "variants": { + "type": "object", "minProperties": 1, + "propertyNames": {"pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}, + "additionalProperties": { + "type": "object", "additionalProperties": false, + "properties": { + "action": {"type": "string", "pattern": "^\\.github/actions/[a-z0-9-]+$"}, + "benchmark_id": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}, + "fresh_inputs": {"type": "object", "additionalProperties": {"type": "string"}}, + "rolling_inputs": {"type": "object", "additionalProperties": {"type": "string"}}, + "environment": {"type": "object", "additionalProperties": {"type": "string"}}, + "provider_flags": {"$ref": "#/$defs/provider_flags"} + } + } + } } }, "blockers": {"type": "array", "items": {"type": "string"}} @@ -86,6 +102,13 @@ "verification": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}} }, "$defs": { + "provider_flags": { + "type": "object", "additionalProperties": false, + "properties": { + "actions-cache": {"type": "object", "additionalProperties": {"enum": ["true", "false"]}}, + "boringcache": {"type": "object", "additionalProperties": {"enum": ["true", "false"]}} + } + }, "sha": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, "path": {"type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))[A-Za-z0-9_./-]+$"} }, diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index 68e682ef0..771f0e7fe 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -122,6 +122,10 @@ def self.validate(cases = documents, root: ROOT) errors << "#{id}: variants differ from the workflow input choices" end end + if File.basename(entry.fetch("path")).start_with?("native-") && item.dig("execution", "native", "variants") + variants = item.dig("execution", "native", "variants").keys + errors << "#{id}: native workflow variants differ from the reviewed recipes" unless entry["variant_input"] == "variant" && entry["variants"]&.sort == variants.sort + end if inputs["cli_version"] && !inputs["cli_version"].fetch("default", "").to_s.empty? errors << "#{id}: cli_version must default to the Action's version" end @@ -208,8 +212,9 @@ def self.plan(item, lane: "fresh", workflow: nil, inputs: {}, variant: nil, root raise Error, "#{name} differs from the declared variant" if combined.key?(name) && combined[name] != variant combined[name] = variant end - if item.dig("execution", "native") && combined["case_id"] != item.fetch("id") - raise Error, "case_id must match the selected case" + if item.dig("execution", "native") && File.basename(entry.fetch("path")).start_with?("native-") + raise Error, "case_id must match the selected case" unless combined["case_id"] == item.fetch("id") + NativeCase.resolve(item.dig("execution", "native"), variant || combined["variant"]) end revisions = item.fetch("source").fetch("pins").map { |pin| pin.fetch("revision") } combined.each do |name, value| @@ -340,7 +345,7 @@ def self.dispatch(item, lane:, workflow: nil, inputs: {}, ref: "main", output:) receipt end - def self.prepare(item, directory:, root: ROOT, native_lane: nil, suffix: "") + def self.prepare(item, directory:, root: ROOT, native_lane: nil, suffix: "", variant: nil) blockers = item.fetch("execution").fetch("blockers", []) raise Error, "#{item.fetch('id')}: #{blockers.join('; ')}" unless blockers.empty? target = File.expand_path(directory) @@ -407,7 +412,7 @@ def self.prepare(item, directory:, root: ROOT, native_lane: nil, suffix: "") "comparison" => item.fetch("comparison"), "execution" => item.fetch("execution")} write_json(File.join(target, "benchmark-context.json"), context) if native_lane - NativeCase.write_action(item, directory: target, lane: native_lane, suffix: suffix) + NativeCase.write_action(item, directory: target, lane: native_lane, suffix: suffix, variant: variant) end target end @@ -459,8 +464,13 @@ def self.resolve_provider_steps(document, provider) def self.write_contract_views(directory, root: ROOT) FileUtils.mkdir_p(directory) raise Error, "Contract destination must be empty" unless Dir.children(directory).empty? - documents(root).reject { |item| item["kind"] == "retained" || !item.dig("execution", "blockers").to_a.empty? }.each do |item| - target = File.join(directory, "benchmark-#{item.fetch('id')}") + runnable = documents(root).reject { |item| item["kind"] == "retained" || !item.dig("execution", "blockers").to_a.empty? } + runnable.flat_map do |item| + variants = item.dig("execution", "native", "variants")&.keys || [nil] + variants.map { |variant| [item, variant] } + end.each do |item, native_variant| + name = "benchmark-#{item.fetch('id')}#{native_variant ? "-#{native_variant}" : ""}" + target = File.join(directory, name) FileUtils.mkdir_p(target) payload = File.join(root, "cases", item.fetch("id"), "payload") if item["adapter"] == "docker" @@ -484,7 +494,7 @@ def self.write_contract_views(directory, root: ROOT) FileUtils.cp(File.join(root, "scripts", "#{name}.rb"), File.join(target, "scripts", "#{name}.rb")) end copy_shared_actions(target, root: root) - NativeCase.write_action(item, directory: target, lane: "fresh") if item.dig("execution", "native") + NativeCase.write_action(item, directory: target, lane: "fresh", variant: native_variant) if item.dig("execution", "native") # Legacy product guards consume the resolved public Action invocation. # Expand the shared wrapper in these disposable views, preserving its # defaults and each caller's phase, adapter, and strict failure settings. @@ -495,9 +505,13 @@ def self.write_contract_views(directory, root: ROOT) next if path == provider_path document = YAML.safe_load(File.read(path), aliases: true) if item.dig("execution", "native") && File.basename(path).start_with?("native-") - recipe = item.dig("execution", "native") + recipe = NativeCase.resolve(item.dig("execution", "native"), native_variant) document["env"]["BENCHMARK_ID"] = "${{ format('#{recipe.fetch('benchmark_id')}{0}', inputs.benchmark_id_suffix) }}" + document["env"]["BENCHMARK_VARIANT_SUFFIX"] = native_variant ? "-#{native_variant}" : "" (document["on"] || document[true])["workflow_dispatch"]["inputs"]["case_id"]["default"] = item.fetch("id") + (document["on"] || document[true]).values.each do |trigger| + trigger["inputs"]["variant"]["default"] = native_variant.to_s if trigger.is_a?(Hash) && trigger.dig("inputs", "variant") + end end resolved = resolve_provider_steps(document, provider) File.write(path, YAML.dump(resolved)) diff --git a/scripts/check-report-contract.rb b/scripts/check-report-contract.rb index 5e42e5316..703c4f296 100755 --- a/scripts/check-report-contract.rb +++ b/scripts/check-report-contract.rb @@ -184,12 +184,13 @@ def matrix_combinations(matrix) combinations end -def action_variant_input(repo_dir, uses) +def action_variant_input(repo_dir, uses, inputs) action_dir = uses.delete_prefix("./") path = ["action.yml", "action.yaml"].map { |basename| File.join(repo_dir, action_dir, basename) }.find { |candidate| File.file?(candidate) } return nil unless path source = File.read(path) + return "report_variant" if inputs.key?("report_variant") && YAML.safe_load(source, aliases: true).fetch("inputs", {}).key?("report_variant") return nil unless source.include?("--variant") return "arch" if source.match?(/--variant\s+["']?\$ARCH\b/) @@ -210,7 +211,7 @@ def producers_for(workflow:, repo_dir:, job_id:, job:) uses = step["uses"].to_s if uses.start_with?("./.github/actions/") with = step["with"].is_a?(Hash) ? step["with"] : {} - variant_input = action_variant_input(repo_dir, uses) + variant_input = action_variant_input(repo_dir, uses, with) variant = workflow.resolve(with[variant_input].to_s, matrix).to_s if variant_input fields = { "benchmark" => workflow.resolve(with["benchmark_id"].to_s, matrix), diff --git a/scripts/native-case.rb b/scripts/native-case.rb index c9f816efe..14d48b000 100644 --- a/scripts/native-case.rb +++ b/scripts/native-case.rb @@ -6,17 +6,44 @@ module NativeCase class Error < StandardError; end - INPUTS = %w[strategy phase cache_lane benchmark_id cli_version buildkit_image cli_platform].freeze + INPUTS = %w[strategy phase cache_lane benchmark_id cli_version buildkit_image cli_platform report_variant].freeze def self.validate(item, payload:) recipe = item.dig("execution", "native") return unless recipe + selected = recipe.fetch("variants", {}).keys + selected = [nil] if selected.empty? + selected.each { |variant| validate_recipe(resolve(recipe, variant), payload: payload, variant: variant) } + raise Error, "Native comparison requires BoringCache and Actions Cache arms" unless item.dig("comparison", "providers").sort == %w[actions-cache boringcache] + recipe + end + + def self.resolve(recipe, variant) + variants = recipe.fetch("variants", {}) + return recipe if variants.empty? && variant.to_s.empty? + raise Error, "This case does not declare native variants" if variants.empty? + raise Error, "Select a native variant: #{variants.keys.join(', ')}" unless variants.key?(variant) + selected = variants.fetch(variant) + resolved = recipe.reject { |key, _| key == "variants" }.merge(selected) + %w[fresh_inputs rolling_inputs environment].each do |key| + resolved[key] = recipe.fetch(key, {}).merge(selected.fetch(key, {})) + end + inherited = recipe.fetch("provider_flags", {}) + overrides = selected.fetch("provider_flags", {}) + resolved["provider_flags"] = (inherited.keys | overrides.keys).to_h do |provider| + [provider, inherited.fetch(provider, {}).merge(overrides.fetch(provider, {}))] + end + resolved + end + + def self.validate_recipe(recipe, payload:, variant:) path = File.join(payload, recipe.fetch("action"), "action.yml") raise Error, "Missing native case action #{recipe.fetch('action')}" unless File.file?(path) action = YAML.safe_load(File.read(path), aliases: true) raise Error, "Native case action must be composite" unless action.dig("runs", "using") == "composite" declared = action.fetch("inputs") + raise Error, "Native variants require the action's report_variant input" if variant && !declared.key?("report_variant") %w[fresh rolling].each do |lane| settings = recipe.fetch("#{lane}_inputs") unknown = settings.keys - declared.keys @@ -26,26 +53,51 @@ def self.validate(item, payload:) missing = required - settings.keys - INPUTS raise Error, "Missing #{lane} action inputs: #{missing.join(', ')}" unless missing.empty? end - raise Error, "Native comparison requires BoringCache and Actions Cache arms" unless item.dig("comparison", "providers").sort == %w[actions-cache boringcache] + recipe.fetch("provider_flags", {}).each do |provider, flags| + raise Error, "Unknown native provider #{provider}" unless %w[actions-cache boringcache].include?(provider) + flags.each do |name, value| + raise Error, "Provider flags cannot change shared runtime inputs" if INPUTS.include?(name) + raise Error, "Provider flags require declared boolean action inputs" unless declared.key?(name) && %w[true false].include?(value) + %w[fresh rolling].each do |lane| + fallback = recipe.fetch("#{lane}_inputs").fetch(name, declared.fetch(name).fetch("default", "")) + raise Error, "Provider flag #{name} has no boolean fallback" unless %w[true false].include?(fallback) + end + end + end recipe end - def self.write_action(item, directory:, lane:, suffix: "") - recipe = validate(item, payload: directory) - raise Error, "This case does not use the shared native comparison" unless recipe + def self.write_action(item, directory:, lane:, suffix: "", variant: nil) + variant = nil if variant.to_s.empty? + base = validate(item, payload: directory) + raise Error, "This case does not use the shared native comparison" unless base + recipe = resolve(base, variant) raise Error, "Use fresh or rolling for the native lane" unless %w[fresh rolling].include?(lane) raise Error, "Use an empty suffix or a lowercase suffix beginning with a hyphen" unless suffix.match?(/\A(?:-[a-z0-9]+(?:-[a-z0-9]+)*)?\z/) benchmark_id = recipe.fetch("benchmark_id") + suffix declared = YAML.safe_load(File.read(File.join(directory, recipe.fetch("action"), "action.yml")), aliases: true).fetch("inputs") settings = recipe.fetch("#{lane}_inputs").merge(INPUTS.to_h { |name| [name, "${{ inputs.#{name} }}"] }.select { |name, _| declared.key?(name) }) settings["cache_lane"] = lane + settings["report_variant"] = variant.to_s if declared.key?("report_variant") + overrides = recipe.fetch("provider_flags", {}) + overrides.values.flat_map(&:keys).uniq.each do |name| + fallback = settings.fetch(name) { declared.fetch(name).fetch("default") } + actions = overrides.fetch("actions-cache", {}).fetch(name, fallback) + boringcache = overrides.fetch("boringcache", {}).fetch(name, fallback) + settings[name] = "${{ inputs.strategy == 'actions-cache' && '#{actions}' || '#{boringcache}' }}" + end document = {"name" => "Declared benchmark phase", "description" => "Executes the reviewed native case recipe.", "inputs" => INPUTS.to_h { |name| [name, {"required" => %w[strategy phase benchmark_id].include?(name), "default" => ""}] }, "runs" => {"using" => "composite", "steps" => [{"uses" => "./#{recipe.fetch('action')}", "with" => settings, "env" => recipe.fetch("environment", {})}]}} path = File.join(directory, ".github", "actions", "benchmark-phase", "action.yml") FileUtils.mkdir_p(File.dirname(path)) File.write(path, YAML.dump(document)) - File.open(ENV.fetch("GITHUB_ENV"), "a") { |file| file.puts("BENCHMARK_ID=#{benchmark_id}") } if ENV["GITHUB_ENV"] + if ENV["GITHUB_ENV"] + File.open(ENV.fetch("GITHUB_ENV"), "a") do |file| + file.puts("BENCHMARK_ID=#{benchmark_id}") + file.puts("BENCHMARK_VARIANT_SUFFIX=#{variant ? "-#{variant}" : ""}") + end + end document end end diff --git a/test/native_case_test.rb b/test/native_case_test.rb index 8bd4f100d..a2c0f8e72 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -38,6 +38,82 @@ def test_shared_workflow_cannot_dispatch_a_different_case assert_raises(BenchmarkCases::Error) { BenchmarkCases.plan(item, inputs: {"case_id" => "storybook"}) } end + def test_workload_variants_select_one_reviewed_recipe_and_report_identity + with_payload("n8n") do |item, directory| + %w[turbo docker runners distroless].each do |variant| + plan = BenchmarkCases.plan(item, variant: variant) + assert_equal variant, plan.dig("inputs", "variant") + assert_equal "native-fresh-benchmark.yml", plan.fetch("workflow") + action = NativeCase.write_action(item, directory: directory, lane: "fresh", variant: variant) + step = action.dig("runs", "steps", 0) + assert_equal variant, step.dig("with", "report_variant") + assert_equal "26.7.0", step.dig("with", "node_version") + if variant == "turbo" + assert_equal "./.github/actions/n8n-turbo-benchmark", step.fetch("uses") + else + assert_equal "./.github/actions/n8n-docker-benchmark", step.fetch("uses") + assert_match(/upstream\/docker\/images\//, step.dig("with", "dockerfile_path")) + end + end + assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh") } + assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh", variant: "unknown") } + assert_raises(BenchmarkCases::Error) { BenchmarkCases.plan(item, variant: "turbo", inputs: {"variant" => "docker"}) } + end + end + + def test_native_variant_recipes_are_validated_before_selection + with_payload("mastodon") do |item, directory| + action = NativeCase.write_action(item, directory: directory, lane: "fresh", variant: "server-sccache") + assert_equal "server", action.dig("runs", "steps", 0, "with", "workload") + assert_equal "${{ inputs.strategy == 'actions-cache' && 'false' || 'true' }}", action.dig("runs", "steps", 0, "with", "docker_tool_cache") + item["execution"]["native"]["variants"]["streaming"]["fresh_inputs"]["workload"] = "${{ inputs.workload }}" + assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh", variant: "server") } + end + end + + def test_n8n_dependency_installation_does_not_increase_build_and_reuse_time + path = File.join(BenchmarkCases::ROOT, "cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml") + steps = YAML.safe_load(File.read(path)).dig("runs", "steps") + install = steps.index { |step| step["name"] == "Install n8n dependencies" } + timer = steps.index { |step| step["id"] == "build_timer" } + assert_operator install, :<, timer + assert_operator timer, :<, steps.index { |step| step["name"] == "Build n8n" } + reporter = steps.find { |step| step["name"] == "Write the benchmark phase evidence" } + assert_equal "${{ steps.setup_timing.outputs.setup_seconds }}", reporter.dig("env", "SETUP_SECONDS") + Dir.mktmpdir("n8n-timing-") do |directory| + clock = File.join(directory, "clock") + output = File.join(directory, "output") + date = File.join(directory, "date") + File.write(date, "#!/bin/sh\ncat \"$CLOCK_FILE\"\n") + File.chmod(0o755, date) + environment = {"PATH" => "#{directory}:#{ENV.fetch('PATH')}", "CLOCK_FILE" => clock, "GITHUB_OUTPUT" => output} + {"setup_timing" => [103, {"SETUP_STARTED_AT" => "100"}], + "build_timer" => [113, {}], "build_timing" => [120, {"BUILD_STARTED_AT" => "113"}]}.each do |id, (time, inputs)| + File.write(clock, time.to_s) + step = steps.find { |entry| entry["id"] == id } + _, errors, status = Open3.capture3(environment.merge(inputs), "bash", "-c", step.fetch("run")) + assert status.success?, errors + end + values = File.readlines(output).to_h { |line| line.strip.split("=", 2) } + assert_equal "3", values.fetch("setup_seconds") + assert_equal "7", values.fetch("build_seconds") + end + end + + def test_provider_flags_preserve_the_comparator_and_cannot_change_phase_identity + with_payload("posthog") do |item, directory| + item["execution"]["native"]["provider_flags"] = {"actions-cache" => {"docker_tool_cache" => "false"}} + item["execution"]["native"]["variants"]["combined"]["provider_flags"] = {"actions-cache" => {"docker_mount_cache" => "false"}} + action = NativeCase.write_action(item, directory: directory, lane: "fresh", variant: "combined") + %w[docker_tool_cache docker_mount_cache].each do |name| + assert_equal "${{ inputs.strategy == 'actions-cache' && 'false' || 'true' }}", action.dig("runs", "steps", 0, "with", name) + end + item["execution"]["native"]["provider_flags"]["actions-cache"]["phase"] = "true" + error = assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh", variant: "combined") } + assert_includes error.message, "cannot change shared runtime inputs" + end + end + def test_unreviewed_recipe_input_and_unsafe_suffix_are_rejected with_payload("hugo-go") do |item, directory| assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh", suffix: "\nOTHER=value") } From b6b4f3a0670561b5fb9f6f29bd0d70fb44389765 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 02:47:22 +0100 Subject: [PATCH 02/17] Record the shared n8n Turbo qualification request --- .../1-native-fresh-benchmark.yml.json | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 results/n8n/shared-turbo-qualification-01/dispatches/1-native-fresh-benchmark.yml.json diff --git a/results/n8n/shared-turbo-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/n8n/shared-turbo-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..d2d40964e --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "turbo", + "series_id": "shared-turbo-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-03T01:46:05Z", + "ref": "native-workload-qualification-2026-10-03", + "state": "requested", + "run_id": 37087350856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856" +} From 2e47184e9862e5f947b862ab57b341c44fbdcce7 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 02:52:59 +0100 Subject: [PATCH 03/17] Declare package read permission for the validation caller --- .github/workflows/guardrails.yml | 1 + docs/process.md | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/guardrails.yml b/.github/workflows/guardrails.yml index b560778ca..8bb761e76 100644 --- a/.github/workflows/guardrails.yml +++ b/.github/workflows/guardrails.yml @@ -77,6 +77,7 @@ jobs: permissions: contents: read actions: read + packages: read id-token: write with: case_id: hugo-go diff --git a/docs/process.md b/docs/process.md index 65688a18d..7509868dc 100644 --- a/docs/process.md +++ b/docs/process.md @@ -75,6 +75,9 @@ shared actions such as `prepare-case`, `setup-docker`, and `setup-node`; case actions retain their upstream commands, patches, and output verification. Native fresh and rolling workflows support both dispatch and `workflow_call`. Use them for evaluations and published cases without copying the provider lifecycle. +Reusable callers declare `contents: read`, `actions: read`, `packages: read`, and +`id-token: write` for the called job. GitHub checks these permissions when loading +the workflow, including calls whose execution condition is false. Docker tool caches remain product-plan settings rather than a second implementation of the product's proxy or cache lifecycle. The harness Ruby version is pinned once in `.tool-versions`. Ruby setup reads From 35e10981f5e27824d81b19c575a91691592905e9 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 07:48:19 +0100 Subject: [PATCH 04/17] Verify native Docker outputs and declare their qualification series --- cases/chroma/case.json | 10 +- .../chroma-docker-benchmark/action.yml | 23 +- cases/chroma/recipe-review.md | 20 + cases/duckgres/case.json | 9 +- .../duckgres-docker-benchmark/action.yml | 21 +- cases/hugo/case.json | 9 +- .../actions/hugo-docker-benchmark/action.yml | 21 +- cases/immich/case.json | 9 +- .../immich-docker-benchmark/action.yml | 23 +- cases/linkerd2/case.json | 9 +- .../linkerd2-docker-benchmark/action.yml | 21 +- cases/mastodon/case.json | 8 +- .../mastodon-docker-benchmark/action.yml | 26 +- cases/n8n/case.json | 14 +- .../actions/n8n-docker-benchmark/action.yml | 23 +- cases/posthog/case.json | 8 +- .../posthog-docker-benchmark/action.yml | 18 +- data/latest/series.json | 624 ++++++++++++++++++ docs/migration.md | 14 + docs/process.md | 18 + .../image-output-qualification-01/series.json | 46 ++ .../image-output-qualification-01/series.json | 45 ++ .../image-output-qualification-01/series.json | 45 ++ .../image-output-qualification-01/series.json | 45 ++ .../image-output-qualification-01/series.json | 45 ++ .../image-output-qualification-01/series.json | 46 ++ .../image-output-qualification-01/series.json | 46 ++ .../image-output-qualification-01/series.json | 46 ++ scripts/activate-docker-plan.rb | 13 +- scripts/benchmark-cases.rb | 10 +- scripts/benchmark-evidence.rb | 14 +- scripts/benchmark-series.rb | 1 + scripts/native-case.rb | 14 + scripts/prepare-source.rb | 3 + scripts/verify-docker-output.rb | 34 + test/benchmark_case_scripts_test.rb | 109 ++- test/benchmark_evidence_test.rb | 29 +- test/benchmark_series_test.rb | 28 +- test/native_case_test.rb | 48 +- 39 files changed, 1536 insertions(+), 59 deletions(-) create mode 100644 cases/chroma/recipe-review.md create mode 100644 results/chroma/image-output-qualification-01/series.json create mode 100644 results/duckgres/image-output-qualification-01/series.json create mode 100644 results/hugo/image-output-qualification-01/series.json create mode 100644 results/immich/image-output-qualification-01/series.json create mode 100644 results/linkerd2/image-output-qualification-01/series.json create mode 100644 results/mastodon/image-output-qualification-01/series.json create mode 100644 results/n8n/image-output-qualification-01/series.json create mode 100644 results/posthog/image-output-qualification-01/series.json create mode 100644 scripts/verify-docker-output.rb diff --git a/cases/chroma/case.json b/cases/chroma/case.json index 2b96c466e..eb47dab4d 100644 --- a/cases/chroma/case.json +++ b/cases/chroma/case.json @@ -47,7 +47,9 @@ "push_image": "true" }, "environment": {}, - "fresh_inputs": {} + "fresh_inputs": { + "load_image": "true" + } }, "cache_tags": { "chroma-docker-local": "{scope}-docker", @@ -61,7 +63,9 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -69,7 +73,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml b/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml index a4c27188f..f5117c27e 100644 --- a/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml +++ b/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml @@ -12,6 +12,9 @@ inputs: push_image: required: false default: 'false' + load_image: + required: false + default: 'false' cli_version: required: false default: '' @@ -37,7 +40,7 @@ runs: shell: bash env: PREPARE_PHASE: "${{ inputs.phase == 'warm' && 'warm1' || 'base' }}" - run: ./scripts/prepare-source.sh "$PREPARE_PHASE" + run: ruby ./scripts/prepare-source.rb "$PREPARE_PHASE" - name: Select the cache cohort id: scope shell: bash @@ -73,10 +76,11 @@ runs: if: inputs.strategy == 'boringcache' shell: bash env: + LOAD_IMAGE: "${{ inputs.load_image }}" PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/chroma-benchmark:boringcache - run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" + run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -114,6 +118,7 @@ runs: platforms: linux/amd64 tags: ghcr.io/${{ github.repository_owner }}/chroma-benchmark:actions-cache push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + load: "${{ inputs.load_image == 'true' }}" build-args: RELEASE_MODE=1 provenance: false cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} @@ -147,9 +152,19 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/chroma-benchmark:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" BENCHMARK_ID: "${{ inputs.benchmark_id }}" STRATEGY: "${{ inputs.strategy }}" CACHE_LANE: "${{ inputs.cache_lane }}" @@ -169,7 +184,9 @@ runs: SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/chroma/recipe-review.md b/cases/chroma/recipe-review.md new file mode 100644 index 000000000..7149b2678 --- /dev/null +++ b/cases/chroma/recipe-review.md @@ -0,0 +1,20 @@ +# Chroma fresh comparison + +The native fresh comparison uses the pinned source and the committed Docker +wrapper for both providers. It builds the CLI target for Linux AMD64. Both arms +export and load the image during the measured build, then inspect that image. +The BoringCache plan retains its declared compiler-cache proof checks. + +Cold and warm phases use the shared source preparer, which restores the exact +gitlink and removes untracked files in the disposable source checkout. Warm is +an identical-source replay. The older case-specific preparer created +`rust/.boringcache-warm-source-change` to invalidate a Docker source layer and +exercise tool-cache reuse. That diagnostic marker is excluded from the new +native comparison. The original script remains as historical recipe material; +it is not called by this action. + +This projection changes both the output export and the warm-source preparation. +It requires a new predeclared series. Older timings cannot be combined with the +new series. Image inspection establishes a produced image; it does not test the +application's runtime behavior. Live output and completion qualification remain +required before publication or caller cutover. diff --git a/cases/duckgres/case.json b/cases/duckgres/case.json index 43232ea17..0e46cc0a7 100644 --- a/cases/duckgres/case.json +++ b/cases/duckgres/case.json @@ -47,7 +47,9 @@ "push_image": "true" }, "environment": {}, - "fresh_inputs": {} + "fresh_inputs": { + "load_image": "true" + } }, "cache_tags": { "duckgres-docker-local": "{scope}-docker" @@ -60,7 +62,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -68,7 +71,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml b/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml index 8dd3d3114..d1d826b86 100644 --- a/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml +++ b/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml @@ -12,6 +12,9 @@ inputs: push_image: required: false default: 'false' + load_image: + required: false + default: 'false' cli_version: required: false default: '' @@ -73,12 +76,13 @@ runs: if: inputs.strategy == 'boringcache' shell: bash env: + LOAD_IMAGE: "${{ inputs.load_image }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/duckgres-benchmark:boringcache run: ruby ./scripts/activate-docker-plan.rb --source-sha "$SOURCE_SHA" --push - "$PUSH_IMAGE" --image "$IMAGE" + "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -115,6 +119,7 @@ runs: platforms: linux/amd64 tags: ghcr.io/${{ github.repository_owner }}/duckgres-benchmark:actions-cache push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + load: "${{ inputs.load_image == 'true' }}" build-args: | VERSION=build-${{ steps.scope.outputs.source_sha }} COMMIT=${{ steps.scope.outputs.source_sha }} @@ -149,9 +154,19 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/duckgres-benchmark:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" BENCHMARK_ID: "${{ inputs.benchmark_id }}" STRATEGY: "${{ inputs.strategy }}" CACHE_LANE: "${{ inputs.cache_lane }}" @@ -170,7 +185,9 @@ runs: SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/hugo/case.json b/cases/hugo/case.json index abeefd0d5..1c483b0f9 100644 --- a/cases/hugo/case.json +++ b/cases/hugo/case.json @@ -47,7 +47,9 @@ "push_image": "true" }, "environment": {}, - "fresh_inputs": {} + "fresh_inputs": { + "load_image": "true" + } }, "cache_tags": { "hugo-docker-local": "{scope}-docker" @@ -60,7 +62,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -68,7 +71,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml b/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml index 2e73f046e..7bac6d2c1 100644 --- a/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml +++ b/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml @@ -12,6 +12,9 @@ inputs: push_image: required: false default: 'false' + load_image: + required: false + default: 'false' cli_version: required: false default: '' @@ -72,10 +75,11 @@ runs: if: inputs.strategy == 'boringcache' shell: bash env: + LOAD_IMAGE: "${{ inputs.load_image }}" PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/hugo-benchmark:boringcache - run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" + run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -111,6 +115,7 @@ runs: file: upstream/Dockerfile tags: ghcr.io/${{ github.repository_owner }}/hugo-benchmark:actions-cache push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + load: "${{ inputs.load_image == 'true' }}" build-args: HUGO_BUILD_TAGS=extended,withdeploy provenance: mode=max sbom: true @@ -145,9 +150,19 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/hugo-benchmark:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" BENCHMARK_ID: "${{ inputs.benchmark_id }}" STRATEGY: "${{ inputs.strategy }}" CACHE_LANE: "${{ inputs.cache_lane }}" @@ -166,7 +181,9 @@ runs: SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/immich/case.json b/cases/immich/case.json index 26a7bde52..b90eee6d2 100644 --- a/cases/immich/case.json +++ b/cases/immich/case.json @@ -49,7 +49,9 @@ "native": { "action": ".github/actions/immich-docker-benchmark", "benchmark_id": "immich-server", - "fresh_inputs": {}, + "fresh_inputs": { + "load_image": "true" + }, "rolling_inputs": {} } }, @@ -60,7 +62,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -68,7 +71,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml b/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml index 830396828..3ab1cd929 100644 --- a/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml +++ b/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml @@ -23,6 +23,9 @@ inputs: required: false default: '' description: Declared series workload variant. + load_image: + required: false + default: 'false' runs: using: composite steps: @@ -80,9 +83,10 @@ runs: PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/immich-server-benchmark:boringcache + LOAD_IMAGE: "${{ inputs.load_image }}" run: ruby ./scripts/activate-docker-plan.rb --scenario server --source-sha "$SOURCE_SHA" --build-id "$BUILD_ID" --source-ref "$SOURCE_REF" --push "$PUSH_IMAGE" --image - "$IMAGE" + "$IMAGE" --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -127,6 +131,7 @@ runs: DEVICE=cpu cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max + load: "${{ inputs.load_image == 'true' }}" - name: Publish with BoringCache id: publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' @@ -154,6 +159,17 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/immich-server-benchmark:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -174,9 +190,12 @@ runs: SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" REPORT_VARIANT: "${{ inputs.report_variant }}" + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/linkerd2/case.json b/cases/linkerd2/case.json index 7b4827eb1..66a8ff1f5 100644 --- a/cases/linkerd2/case.json +++ b/cases/linkerd2/case.json @@ -47,7 +47,9 @@ "push_image": "true" }, "environment": {}, - "fresh_inputs": {} + "fresh_inputs": { + "load_image": "true" + } }, "cache_tags": { "linkerd2-docker-local": "{scope}-docker" @@ -60,7 +62,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -68,7 +71,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml b/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml index b01cba011..886a1dc16 100644 --- a/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml +++ b/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml @@ -12,6 +12,9 @@ inputs: push_image: required: false default: 'false' + load_image: + required: false + default: 'false' cli_version: required: false default: '' @@ -70,12 +73,13 @@ runs: if: inputs.strategy == 'boringcache' shell: bash env: + LOAD_IMAGE: "${{ inputs.load_image }}" SOURCE_TAG: "${{ steps.scope.outputs.source_tag }}" PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/linkerd2-benchmark:boringcache run: ruby ./scripts/activate-docker-plan.rb --source-tag "$SOURCE_TAG" --push - "$PUSH_IMAGE" --image "$IMAGE" + "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -111,6 +115,7 @@ runs: file: upstream/web/Dockerfile tags: ghcr.io/${{ github.repository_owner }}/linkerd2-benchmark:actions-cache push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + load: "${{ inputs.load_image == 'true' }}" build-args: LINKERD_VERSION=${{ steps.scope.outputs.source_tag }} provenance: false cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} @@ -142,9 +147,19 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/linkerd2-benchmark:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" BENCHMARK_ID: "${{ inputs.benchmark_id }}" STRATEGY: "${{ inputs.strategy }}" CACHE_LANE: "${{ inputs.cache_lane }}" @@ -163,7 +178,9 @@ runs: SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/mastodon/case.json b/cases/mastodon/case.json index a11ac481d..520cc35cb 100644 --- a/cases/mastodon/case.json +++ b/cases/mastodon/case.json @@ -56,7 +56,8 @@ "action": ".github/actions/mastodon-docker-benchmark", "benchmark_id": "mastodon", "fresh_inputs": { - "workload": "server" + "workload": "server", + "load_image": "true" }, "rolling_inputs": { "workload": "server" @@ -97,7 +98,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -105,7 +107,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml b/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml index 1ed45d00b..c57cb168a 100644 --- a/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml +++ b/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml @@ -31,6 +31,9 @@ inputs: required: false default: '' description: Declared series workload variant. + load_image: + required: false + default: 'false' runs: using: composite steps: @@ -57,7 +60,7 @@ runs: shell: bash env: PREPARE_PHASE: "${{ inputs.phase == 'warm' && 'warm1' || 'base' }}" - run: ./scripts/prepare-source.sh "$PREPARE_PHASE" + run: ruby ./scripts/prepare-source.rb "$PREPARE_PHASE" - name: Select the cache cohort id: scope shell: bash @@ -98,9 +101,10 @@ runs: }}" IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.workload == 'server' && 'mastodon-benchmark' || 'mastodon-streaming-benchmark' }}:boringcache + LOAD_IMAGE: "${{ inputs.load_image }}" run: ruby ./scripts/activate-docker-plan.rb --workload "$WORKLOAD" --tool-cache "$TOOL_CACHE" --source-sha "$SOURCE_SHA" --prerelease "$PRERELEASE" --push "$PUSH_IMAGE" - --image "$IMAGE" + --image "$IMAGE" --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -147,6 +151,7 @@ runs: provenance: false cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max + load: "${{ inputs.load_image == 'true' }}" - name: Publish with BoringCache id: publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' @@ -174,6 +179,18 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.workload == 'server' + && 'mastodon-benchmark' || 'mastodon-streaming-benchmark' }}:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -194,9 +211,12 @@ runs: SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" REPORT_VARIANT: "${{ inputs.report_variant }}" + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/n8n/case.json b/cases/n8n/case.json index 0ed6fad54..e06ef7df5 100644 --- a/cases/n8n/case.json +++ b/cases/n8n/case.json @@ -79,7 +79,8 @@ "benchmark_id": "n8n-docker", "fresh_inputs": { "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", - "image": "n8n-benchmark" + "image": "n8n-benchmark", + "load_image": "true" }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", @@ -91,7 +92,8 @@ "benchmark_id": "n8n-runners", "fresh_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile", - "image": "n8n-runners-benchmark" + "image": "n8n-runners-benchmark", + "load_image": "true" }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile", @@ -103,7 +105,8 @@ "benchmark_id": "n8n-runners-distroless", "fresh_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", - "image": "n8n-runners-distroless-benchmark" + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", @@ -123,7 +126,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -131,7 +135,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml b/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml index 8f89f1762..bab5e0aae 100644 --- a/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml +++ b/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml @@ -29,6 +29,9 @@ inputs: required: false default: '' description: Declared series workload variant. + load_image: + required: false + default: 'false' runs: using: composite steps: @@ -101,8 +104,9 @@ runs: PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.image }}:boringcache + LOAD_IMAGE: "${{ inputs.load_image }}" run: ruby ./scripts/activate-docker-plan.rb --dockerfile "$DOCKERFILE" --node-version - "$NODE_VERSION" --push "$PUSH_IMAGE" --image "$IMAGE" + "$NODE_VERSION" --push "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" - name: Set up Node.js uses: "./.github/actions/setup-node" with: @@ -174,6 +178,7 @@ runs: sbom: false cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max + load: "${{ inputs.load_image == 'true' }}" - name: Publish with BoringCache id: publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' @@ -203,6 +208,17 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.image }}:actions-cache + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -223,9 +239,12 @@ runs: SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" REPORT_VARIANT: "${{ inputs.report_variant }}" + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/cases/posthog/case.json b/cases/posthog/case.json index 003bf7c17..33caad24a 100644 --- a/cases/posthog/case.json +++ b/cases/posthog/case.json @@ -55,7 +55,8 @@ "action": ".github/actions/posthog-docker-benchmark", "benchmark_id": "posthog", "fresh_inputs": { - "platform": "linux/amd64" + "platform": "linux/amd64", + "load_image": "true" }, "rolling_inputs": { "platform": "linux/amd64" @@ -89,7 +90,8 @@ "publication": "reviewed" }, "verification": [ - "Pinned upstream recipe and build success" + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" ], "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -97,7 +99,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml index 922e1e238..36db92e63 100644 --- a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml +++ b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml @@ -280,6 +280,19 @@ runs: env: BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + - name: Verify the Docker output + id: output_verification + shell: bash + env: + STRATEGY: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + LOAD_IMAGE: "${{ inputs.load_image }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" + IMAGE: ghcr.io/${{ github.repository_owner }}/posthog-benchmark:${{ inputs.image_tag + || 'actions-cache' }} + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -300,9 +313,12 @@ runs: GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" + OUTPUT_VERIFIED: "${{ steps.output_verification.outputs.verified }}" run: | set -euo pipefail - ruby ./scripts/benchmark-report.rb phase \ + verification=() + [[ "$OUTPUT_VERIFIED" != 'true' ]] || verification=(--verified-output) + ruby ./scripts/benchmark-report.rb phase "${verification[@]}" \ --benchmark "$BENCHMARK_ID" \ --strategy "$STRATEGY" \ --lane "$CACHE_LANE" \ diff --git a/data/latest/series.json b/data/latest/series.json index 60eb1c8e7..9f24d91e5 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -3,6 +3,73 @@ "repository": "boringcache/benchmarks", "scope": "Canonical series; historical index and reviewed website selections remain separate", "series": [ + { + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "plan_path": "results/chroma/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "deno", "series_id": "consolidated-cargo-proof", @@ -1055,6 +1122,73 @@ ], "runs_without_archives": [] }, + { + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "plan_path": "results/duckgres/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "hugo-go", "series_id": "direct-dispatch-qualification", @@ -1583,6 +1717,429 @@ ], "runs_without_archives": [] }, + { + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "plan_path": "results/hugo/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700", + "definition_sha256": "200bdce61b6cbba1361be02b435abbe9c4750e77dd72058281ba1c1e8a8882f4", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "immich", + "series_id": "image-output-qualification-01", + "plan_path": "results/immich/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "plan_path": "results/linkerd2/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "plan_path": "results/mastodon/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": "server", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "plan_path": "results/n8n/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": "docker", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", + "plan_path": "results/n8n/shared-turbo-qualification-01/series.json", + "lane": "fresh", + "variant": "turbo", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "035df772551717a7a2d369d39994c744f270815a5481833e7a494a06097dcae9", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "turbo", + "series_id": "shared-turbo-qualification-01", + "sample": "1" + }, + "ref": "native-workload-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T01:46:05Z", + "run_id": 37087350856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + } + ], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [ + "37087350856" + ] + }, { "case_id": "obs-studio", "series_id": "direct-ccache-qualification", @@ -2144,6 +2701,73 @@ ], "runs_without_archives": [] }, + { + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "plan_path": "results/posthog/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": "layers", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "zed", "series_id": "direct-combined-qualification", diff --git a/docs/migration.md b/docs/migration.md index 23d42d353..bf450c2f0 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -40,6 +40,20 @@ remain registered diagnostics while output checks and live execution are qualified. The new Docker paths do not establish a loaded or published image, or qualify the original rolling publication behavior. Existing callers and schedules still use the original repositories. +Four initial screens (Chroma, Duckgres, Hugo Docker, and Linkerd2) were cancelled +after review found missing image output checks. Their original plans, receipts, +cancelled completions, and scoped evidence remain preserved. Their replacement +fresh recipes load an image in both arms and use the shared output checker; +timing includes export and load. Those replacement definitions still require +live qualification. Original rolling publication projections remain diagnostic. +The same checker and load projection apply to the selected n8n Docker, Mastodon, +PostHog, and Immich server recipes. Native preparation enforces the output +requirement before creating a source checkout. No image load or application +behavior is claimed for their older cache-only results. +The native Docker actions also use the shared Ruby source preparer. Chroma's +[recipe review](../cases/chroma/recipe-review.md) records the removal of its old +diagnostic source-layer marker from the identical-source replay. The replacement +timings must not be combined with that earlier projection. n8n Turbo's primary measurement now captures cache setup and the build separately from its dependency installation. The install still runs before the build. Earlier results retain their original timed scope and must not be mixed with this series. diff --git a/docs/process.md b/docs/process.md index 7509868dc..f4edb3107 100644 --- a/docs/process.md +++ b/docs/process.md @@ -60,6 +60,19 @@ All variants use the same provider lifecycle and canonical reporter. Missing or unknown variants fail before dispatch. Contract checks prepare every registered native variant, including its report filenames. A fresh variant passing does not qualify another variant or the original rolling matrix. +Native planning also rejects a phase action that does not report verified output. +Add and review the actual output check before setting `--verified-output`; the +flag alone does not verify a workload. Schema and file-contract checks can pass +while an imported recipe still lacks that execution requirement. +For native Docker comparisons that declare `load_image`, fresh inputs must set +it to `true`. Both arms then export and load their built image during the timed +build. The shared `verify-docker-output.rb` checks the selected provider's image +after timing and passes verification to the reporter only after inspection +succeeds. The image check establishes that the build produced the declared image; +application behavior requires any additional checks declared by the case. +Publication and cache-only projections remain diagnostic until their output and +comparison boundary have been reviewed. Do not add the verification flag to a +cache-only build. Keep different provider sets, architectures, source-change sequences, and output behavior explicit in their adapters or case actions. OBS, Cargo product proofs, @@ -231,6 +244,11 @@ Publish the verified bundle to durable storage with a stable link. Experimental cache retention is not permanent evidence retention. The exporter includes all available attempts and records gaps; partial exports remain partial. Keep original execution URLs as provenance. +Cancelled requests remain in their original series. A run cancelled before any +jobs start can have a valid empty log ZIP and a complete scoped export, with no +measurements. Empty logs for a run that has jobs remain an evidence gap. Use a +new series after correcting an execution contract; do not replace the old request +or infer phase measurements from its logs. `finish` checks the preserved run and job conclusions and post-step logs. A green job with a reported BoringCache post-step failure remains unqualified. Reports require a completion check for every run represented by their phase diff --git a/results/chroma/image-output-qualification-01/series.json b/results/chroma/image-output-qualification-01/series.json new file mode 100644 index 000000000..a590a6ce7 --- /dev/null +++ b/results/chroma/image-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:28Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "variant": null, + "workflow_inputs": { + "case_id": "chroma", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7" +} diff --git a/results/duckgres/image-output-qualification-01/series.json b/results/duckgres/image-output-qualification-01/series.json new file mode 100644 index 000000000..fd332f1ec --- /dev/null +++ b/results/duckgres/image-output-qualification-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:28Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "variant": null, + "workflow_inputs": { + "case_id": "duckgres", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20" +} diff --git a/results/hugo/image-output-qualification-01/series.json b/results/hugo/image-output-qualification-01/series.json new file mode 100644 index 000000000..ab3940476 --- /dev/null +++ b/results/hugo/image-output-qualification-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:28Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "200bdce61b6cbba1361be02b435abbe9c4750e77dd72058281ba1c1e8a8882f4", + "variant": null, + "workflow_inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700" +} diff --git a/results/immich/image-output-qualification-01/series.json b/results/immich/image-output-qualification-01/series.json new file mode 100644 index 000000000..2a80a0cd8 --- /dev/null +++ b/results/immich/image-output-qualification-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:29Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "variant": null, + "workflow_inputs": { + "case_id": "immich", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f" +} diff --git a/results/linkerd2/image-output-qualification-01/series.json b/results/linkerd2/image-output-qualification-01/series.json new file mode 100644 index 000000000..eee11b4df --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:28Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "variant": null, + "workflow_inputs": { + "case_id": "linkerd2", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a" +} diff --git a/results/mastodon/image-output-qualification-01/series.json b/results/mastodon/image-output-qualification-01/series.json new file mode 100644 index 000000000..647bda4d8 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:29Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "variant": "server", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "server" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413" +} diff --git a/results/n8n/image-output-qualification-01/series.json b/results/n8n/image-output-qualification-01/series.json new file mode 100644 index 000000000..ef7bff568 --- /dev/null +++ b/results/n8n/image-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:28Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "variant": "docker", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "docker" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5" +} diff --git a/results/posthog/image-output-qualification-01/series.json b/results/posthog/image-output-qualification-01/series.json new file mode 100644 index 000000000..43d41d17f --- /dev/null +++ b/results/posthog/image-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "created_at": "2026-10-03T02:46:29Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2" +} diff --git a/scripts/activate-docker-plan.rb b/scripts/activate-docker-plan.rb index 90f39fe73..805ab458a 100755 --- a/scripts/activate-docker-plan.rb +++ b/scripts/activate-docker-plan.rb @@ -5,10 +5,13 @@ args = {"plan" => File.join(BenchmarkPlan::ROOT, ".boringcache.toml"), "load" => "false", "tool_cache" => "false"} OptionParser.new do |parser| - %w[push image plan source_tag source_sha workload tool_cache prerelease dockerfile node_version scenario build_id source_ref platform mount_cache no_cache sourcemap_secret load].each do |name| + %w[push image plan source_tag source_sha workload tool_cache prerelease dockerfile node_version scenario build_id source_ref platform mount_cache sourcemap_secret load].each do |name| parser.on("--#{name.tr('_', '-')} VALUE") { |value| args[name] = value } end + # The non-negated name makes OptionParser consume the value for --no-cache. + parser.on("--no-cache VALUE", "--disable-layer-cache VALUE", String) { |value| args["no_cache"] = value.to_s } end.parse! +raise "Unexpected arguments: #{ARGV.join(' ')}" unless ARGV.empty? case_id = JSON.parse(File.read(File.join(BenchmarkPlan::ROOT, "benchmark-context.json"))).fetch("case_id") path = args.fetch("plan") text = File.read(path) @@ -23,6 +26,8 @@ value == "true" end push = boolean.call("push") +load = boolean.call("load") +raise "Choose either push or load" if push && load raise "Publishing requires --image" if push && args.fetch("image", "").empty? sha = args["source_sha"] raise "Source SHA must be a full lowercase commit SHA" if sha && !sha.match?(/\A[0-9a-f]{40}\z/) @@ -46,6 +51,7 @@ image = case_id == "linkerd2" ? "linkerd2-web" : case_id replace.call(" \"--tag\", \"#{image}-benchmark:local\",\n \"upstream\",", " \"--tag\", #{JSON.generate(args.fetch('image'))},\n \"--push\",\n \"upstream\",") end + replace.call(" \"upstream\",", " \"--load\",\n \"upstream\",") if load when "mastodon" workload = args.fetch("workload") raise "Workload must be server or streaming" unless %w[server streaming].include?(workload) @@ -63,6 +69,7 @@ replace.call(needle, needle + "\ntool-cache = [\"sccache\"]") end replace.call(" \"upstream\",\n]", " \"--push\",\n \"upstream\",\n]") if push + replace.call(" \"upstream\",\n]", " \"--load\",\n \"upstream\",\n]") if load when "posthog", "immich" docker = plan.fetch("adapters").fetch("docker") if case_id == "posthog" @@ -71,8 +78,6 @@ raise "Expected Dockerfile #{expected}" unless args.fetch("dockerfile") == expected platform = args.fetch("platform") raise "Unsupported platform" unless %w[linux/amd64 linux/arm64].include?(platform) - load = boolean.call("load") - raise "Choose either push or load" if push && load command = ["docker", "buildx", "build", "--file", expected, "--platform", platform, "--provenance", "false"] command += ["--secret", "id=posthog_upload_sourcemaps_cli_api_key,env=POSTHOG_SOURCEMAP_API_KEY"] if boolean.call("sourcemap_secret") command << "--no-cache" if boolean.call("no_cache") @@ -92,6 +97,7 @@ "--build-arg", "BUILD_SOURCE_REF=#{args['source_ref']}", "--build-arg", "BUILD_SOURCE_COMMIT=#{sha}", "--build-arg", "DEVICE=cpu", "--tag", push ? args.fetch("image") : "immich-server-benchmark:local"] command << "--push" if push + command << "--load" if load command << "upstream" docker["metadata-hints"] = ["benchmark=immich-server", "upstream-job=server-amd64"] docker.delete("tool-cache") @@ -99,6 +105,7 @@ raise "Base-images plans do not publish an image" if push command = ["docker", "buildx", "build", "--file", "base-images-upstream/server/Dockerfile", "--platform", "linux/amd64", "--target", "libvips", "--tag", "immich-base-images-benchmark:local", "base-images-upstream/server"] + command.insert(-2, "--load") if load docker["metadata-hints"] = ["benchmark=immich-base-images", "upstream-job=server-native-amd64"] boolean.call("tool_cache") ? docker["tool-cache"] = ["ccache"] : docker.delete("tool-cache") else diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index 771f0e7fe..76bd33bb7 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -22,7 +22,7 @@ module BenchmarkCases ROOT = File.expand_path("..", __dir__) WORKSPACE = "boringcache/benchmarks" REPOSITORY = "boringcache/benchmarks" - HELPERS = %w[benchmark-plan run-benchmark-plan activate-docker-plan summarize-cargo-evidence summarize-sccache-errors docker-case-contract measure-build native-case prepare-source scope-case-cache].freeze + HELPERS = %w[benchmark-plan run-benchmark-plan activate-docker-plan verify-docker-output summarize-cargo-evidence summarize-sccache-errors docker-case-contract measure-build native-case prepare-source scope-case-cache].freeze class Error < StandardError; end def self.command(*args, chdir: nil, stdin: "", env: {}) @@ -214,7 +214,8 @@ def self.plan(item, lane: "fresh", workflow: nil, inputs: {}, variant: nil, root end if item.dig("execution", "native") && File.basename(entry.fetch("path")).start_with?("native-") raise Error, "case_id must match the selected case" unless combined["case_id"] == item.fetch("id") - NativeCase.resolve(item.dig("execution", "native"), variant || combined["variant"]) + recipe = NativeCase.resolve(item.dig("execution", "native"), variant || combined["variant"]) + NativeCase.verify_report(recipe, payload: File.join(root, "cases", item.fetch("id"), "payload"), lane: entry.fetch("lane")) end revisions = item.fetch("source").fetch("pins").map { |pin| pin.fetch("revision") } combined.each do |name, value| @@ -348,6 +349,11 @@ def self.dispatch(item, lane:, workflow: nil, inputs: {}, ref: "main", output:) def self.prepare(item, directory:, root: ROOT, native_lane: nil, suffix: "", variant: nil) blockers = item.fetch("execution").fetch("blockers", []) raise Error, "#{item.fetch('id')}: #{blockers.join('; ')}" unless blockers.empty? + if native_lane + native = item.dig("execution", "native") or raise Error, "This case does not use the shared native comparison" + recipe = NativeCase.resolve(native, variant) + NativeCase.verify_report(recipe, payload: File.join(root, "cases", item.fetch("id"), "payload"), lane: native_lane) + end target = File.expand_path(directory) entries = Dir.exist?(target) ? Dir.children(target) : [] raise Error, "Prepare requires an empty disposable directory or only the .harness checkout" unless (entries - [".harness"]).empty? diff --git a/scripts/benchmark-evidence.rb b/scripts/benchmark-evidence.rb index 6740a4235..4cb1095df 100644 --- a/scripts/benchmark-evidence.rb +++ b/scripts/benchmark-evidence.rb @@ -25,12 +25,21 @@ def download(path, target) end raise BenchmarkCases::Error, "GitHub returned an empty download for #{path}" if File.size(target).zero? if target.end_with?(".zip") - BenchmarkCases.command("unzip", "-tqq", target) + BenchmarkEvidence.verify_zip(target) end File.chmod(0o600, target) end end + def self.empty_zip?(path) + File.size(path) == 22 && File.binread(path) == "PK\x05\x06" + "\x00" * 18 + end + + def self.verify_zip(path) + return if empty_zip?(path) + BenchmarkCases.command("unzip", "-tqq", path) + end + def self.export(repository:, run_id:, directory:, github: GitHub.new) raise BenchmarkCases::Error, "Invalid repository" unless repository.match?(%r{\A[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\z}) raise BenchmarkCases::Error, "Invalid run ID" unless run_id.is_a?(Integer) && run_id.positive? @@ -55,6 +64,9 @@ def self.export(repository:, run_id:, directory:, github: GitHub.new) retain_json(directory, "#{prefix}/jobs.json", {"total_count" => jobs.length, "jobs" => jobs}, manifest) manifest["attempts"] << {"attempt" => attempt, "job_ids" => jobs.map { |job| job.fetch("id") }} retain_download(directory, "#{prefix}/logs.zip", "#{attempt_path}/logs", github, manifest) + if jobs.any? && manifest.fetch("files").key?("#{prefix}/logs.zip") && empty_zip?(File.join(directory, "#{prefix}/logs.zip")) + manifest["gaps"] << {"resource" => "#{prefix}/logs.zip", "reason" => "Jobs exist but the log archive has no entries"} + end rescue BenchmarkCases::Error => error manifest["gaps"] << {"resource" => "#{prefix}/metadata", "reason" => error.message} end diff --git a/scripts/benchmark-series.rb b/scripts/benchmark-series.rb index 7e68bad11..7744e2429 100644 --- a/scripts/benchmark-series.rb +++ b/scripts/benchmark-series.rb @@ -126,6 +126,7 @@ def self.finish(directory, evidence_directory:, sample:) jobs.reject { |job| %w[success skipped].include?(job["conclusion"]) }.each do |job| errors << "#{job.fetch('name')}: #{job['conclusion'] || job['status']}" end + next if jobs.empty? Open3.popen2e("unzip", "-p", File.join(prefix, "logs.zip")) do |input, output, process| input.close output.each_line do |line| diff --git a/scripts/native-case.rb b/scripts/native-case.rb index 14d48b000..1f2d3572e 100644 --- a/scripts/native-case.rb +++ b/scripts/native-case.rb @@ -67,6 +67,20 @@ def self.validate_recipe(recipe, payload:, variant:) recipe end + def self.verify_report(recipe, payload:, lane:) + action = YAML.safe_load(File.read(File.join(payload, recipe.fetch("action"), "action.yml")), aliases: true) + reports = action.dig("runs", "steps").filter_map do |step| + command = step["run"].to_s + command if command.match?(/benchmark-report\.rb\s+phase\b/) + end + unless !reports.empty? && reports.all? { |command| command.match?(/(?:\A|[\s(])--verified-output(?=[\s)]|\z)/) } + raise Error, "Selected native action does not report verified output; add and review its output check before execution" + end + if action.fetch("inputs").key?("load_image") && recipe.fetch("#{lane}_inputs").fetch("load_image", "false") != "true" + raise Error, "Native Docker comparison requires load_image=true; publication and cache-only lanes remain diagnostic" + end + end + def self.write_action(item, directory:, lane:, suffix: "", variant: nil) variant = nil if variant.to_s.empty? base = validate(item, payload: directory) diff --git a/scripts/prepare-source.rb b/scripts/prepare-source.rb index 48893f704..0a652fe5f 100644 --- a/scripts/prepare-source.rb +++ b/scripts/prepare-source.rb @@ -10,6 +10,9 @@ abort "Source preparation requires a prepared case" unless File.file?(File.join(root, "benchmark-context.json")) index, error, status = Open3.capture3("git", "ls-files", "--stage", "upstream", chdir: root) abort "Source preparation requires the declared upstream gitlink: #{error.strip}" unless status.success? && index.match?(/\A160000 [0-9a-f]{40} 0\tupstream\n\z/) +expected = index.split.fetch(1) +actual, error, status = Open3.capture3("git", "-C", File.join(root, "upstream"), "rev-parse", "HEAD") +abort "Upstream HEAD differs from the declared source pin: #{error.strip}" unless status.success? && actual.strip == expected %w[reset clean].each do |operation| arguments = operation == "reset" ? ["reset", "--hard"] : ["clean", "-fdx"] abort "Cannot #{operation} the disposable upstream checkout" unless system("git", "-C", File.join(root, "upstream"), *arguments) diff --git a/scripts/verify-docker-output.rb b/scripts/verify-docker-output.rb new file mode 100644 index 000000000..776cb7f66 --- /dev/null +++ b/scripts/verify-docker-output.rb @@ -0,0 +1,34 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "benchmark-plan" + +options = {"load" => "false", "push" => "false"} +OptionParser.new do |parser| + %w[image strategy load push].each do |name| + parser.on("--#{name} VALUE") { |value| options[name] = value } + end +end.parse! +raise "Unexpected arguments" unless ARGV.empty? +%w[load push].each do |name| + raise "--#{name} must be true or false" unless %w[true false].include?(options.fetch(name)) +end +raise "Choose either push or load" if options["load"] == "true" && options["push"] == "true" +strategy = options.fetch("strategy") +raise "Unknown Docker provider" unless %w[boringcache actions-cache].include?(strategy) + +verified = options["load"] == "true" || options["push"] == "true" +if verified + image = if strategy == "boringcache" + command = BenchmarkPlan.command("docker") + index = command.index("--tag") or raise "Docker plan has no output tag" + command.fetch(index + 1) + else + options.fetch("image") + end + raise "Output image is empty" if image.empty? + command = options["load"] == "true" ? ["docker", "image", "inspect", image] : ["docker", "buildx", "imagetools", "inspect", image] + _output, errors, status = Open3.capture3(*command) + raise "Cannot verify Docker output #{image}: #{errors}" unless status.success? +end +BenchmarkPlan.write_output("verified", verified) diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 79702ab3a..e5d0b4e18 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -31,7 +31,7 @@ def with_case(id) Dir.mktmpdir("benchmark-recipe-") do |directory| FileUtils.cp_r(Dir[File.join(ROOT, "cases", id, "payload", "{*,.[!.]*}")], directory) FileUtils.mkdir_p(File.join(directory, "scripts")) - %w[benchmark-plan activate-docker-plan run-benchmark-plan verify-upstream-recipe scope-case-cache prepare-source].each do |name| + %w[benchmark-plan activate-docker-plan verify-docker-output run-benchmark-plan verify-upstream-recipe scope-case-cache prepare-source].each do |name| FileUtils.cp(File.join(ROOT, "scripts", "#{name}.rb"), File.join(directory, "scripts", "#{name}.rb")) end item = JSON.parse(File.read(File.join(ROOT, "cases", id, "case.json"))) @@ -44,6 +44,113 @@ def run_script(directory, name, *args) Open3.capture3(RbConfig.ruby, File.join(directory, "scripts", "#{name}.rb"), *args, chdir: directory) end + def test_docker_output_projection_loads_each_native_image_without_publication + %w[chroma duckgres hugo linkerd2 n8n mastodon posthog immich].each do |id| + with_case(id) do |directory| + action = YAML.safe_load(File.read(File.join(directory, ".github", "actions", "#{id}-docker-benchmark", "action.yml"))) + activate = action.dig("runs", "steps").find { |step| step["name"] == "Activate the BoringCache Docker plan" } + environment = {"PATH" => "#{File.dirname(RbConfig.ruby)}:#{ENV.fetch('PATH')}", + "LOAD_IMAGE" => "true", "PUSH_IMAGE" => "false", "IMAGE" => "example/image:local", "SOURCE_SHA" => "a" * 40, "SOURCE_TAG" => "test", + "DOCKERFILE" => id == "n8n" ? "upstream/docker/images/n8n/Dockerfile" : "upstream/Dockerfile", "NODE_VERSION" => "26.7.0", + "WORKLOAD" => "server", "TOOL_CACHE" => "false", "PRERELEASE" => "nightly.2026-10-03", "PLATFORM" => "linux/amd64", + "MOUNT_CACHE" => "false", "NO_CACHE" => "false", "SOURCEMAP_SECRET" => "false", "BUILD_ID" => "123", "SOURCE_REF" => "main"} + _, errors, status = Open3.capture3(environment, "bash", "-c", activate.fetch("run"), chdir: directory) + assert status.success?, errors + command = TomlRB.load_file(File.join(directory, ".boringcache.toml")).dig("adapters", "docker", "command") + assert_includes command, "--load" + refute_includes command, "--push" + before = File.read(File.join(directory, ".boringcache.toml")) + _, errors, status = run_script(directory, "activate-docker-plan", "--load", "true", "--push", "true", "--image", "example/image") + refute status.success? + assert_includes errors, "Choose either push or load" + assert_equal before, File.read(File.join(directory, ".boringcache.toml")) + end + end + end + + def test_docker_output_check_uses_each_provider_image_and_rejects_failed_inspection + with_case("hugo") do |directory| + executable = File.join(directory, "docker") + File.write(executable, "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$DOCKER_ARGUMENTS\"\nexit \"$INSPECT_STATUS\"\n") + File.chmod(0o755, executable) + arguments = File.join(directory, "docker-arguments") + output = File.join(directory, "outputs") + environment = {"PATH" => "#{directory}:#{ENV.fetch('PATH')}", "DOCKER_ARGUMENTS" => arguments, + "INSPECT_STATUS" => "0", "GITHUB_OUTPUT" => output} + script = File.join(directory, "scripts", "verify-docker-output.rb") + %w[boringcache actions-cache].each do |provider| + _, errors, status = Open3.capture3(environment, RbConfig.ruby, script, "--strategy", provider, "--load", "true", "--image", "example/actions:local") + assert status.success?, errors + expected = provider == "boringcache" ? "hugo-benchmark:local" : "example/actions:local" + assert_equal ["image", "inspect", expected], File.readlines(arguments, chomp: true) + assert_includes File.read(output), "verified=true" + end + File.delete(arguments) + File.delete(output) + _, errors, status = Open3.capture3(environment, RbConfig.ruby, script, "--strategy", "actions-cache", "--load", "false", "--push", "false") + assert status.success?, errors + refute File.exist?(arguments) + assert_equal "verified=false\n", File.read(output) + File.delete(output) + _, errors, status = Open3.capture3(environment.merge("INSPECT_STATUS" => "1"), RbConfig.ruby, script, + "--strategy", "actions-cache", "--load", "true", "--image", "example/missing:local") + refute status.success? + assert_includes errors, "Cannot verify Docker output" + refute File.exist?(output) + end + end + + def test_posthog_no_cache_option_preserves_its_declared_boolean_value + with_case("posthog") do |directory| + %w[false true].each do |value| + _, errors, status = run_script(directory, "activate-docker-plan", "--dockerfile", "upstream/Dockerfile", + "--platform", "linux/amd64", "--no-cache", value) + assert status.success?, errors + command = TomlRB.load_file(File.join(directory, ".boringcache.toml")).dig("adapters", "docker", "command") + assert_equal value == "true", command.include?("--no-cache") + end + before = File.read(File.join(directory, ".boringcache.toml")) + _, _, status = run_script(directory, "activate-docker-plan", "--no-cache", "unknown") + refute status.success? + assert_equal before, File.read(File.join(directory, ".boringcache.toml")) + end + end + + def test_clean_source_replay_checks_the_pin_before_removing_untracked_files + with_case("hugo") do |directory| + source = File.join(directory, "upstream") + FileUtils.mkdir_p(source) + [directory, source].each do |path| + _, errors, status = Open3.capture3("git", "init", path) + assert status.success?, errors + end + File.write(File.join(source, "source.txt"), "pinned source\n") + _, errors, status = Open3.capture3("git", "-C", source, "add", "source.txt") + assert status.success?, errors + _, errors, status = Open3.capture3("git", "-C", source, "-c", "user.name=Benchmark test", "-c", "user.email=test@localhost", + "-c", "commit.gpgsign=false", "commit", "-m", "Pin source") + assert status.success?, errors + sha, _, status = Open3.capture3("git", "-C", source, "rev-parse", "HEAD") + assert status.success? + untracked = File.join(source, "untracked.txt") + File.write(untracked, "retain on mismatched pin\n") + ["a" * 40, sha.strip].each do |pin| + _, errors, status = Open3.capture3("git", "-C", directory, "update-index", "--add", "--cacheinfo", "160000,#{pin},upstream") + assert status.success?, errors + _, errors, status = run_script(directory, "prepare-source", "warm1") + if pin == sha.strip + assert status.success?, errors + refute File.exist?(untracked) + assert_equal "pinned source\n", File.read(File.join(source, "source.txt")) + else + refute status.success? + assert_includes errors, "differs from the declared source pin" + assert File.exist?(untracked) + end + end + end + end + def test_mastodon_streaming_does_not_acquire_server_tool_cache with_case("mastodon") do |directory| _, error, status = run_script(directory, "activate-docker-plan", "--workload", "streaming", "--tool-cache", "false", diff --git a/test/benchmark_evidence_test.rb b/test/benchmark_evidence_test.rb index 5ddbf4fb7..a4d3a2594 100644 --- a/test/benchmark_evidence_test.rb +++ b/test/benchmark_evidence_test.rb @@ -9,10 +9,12 @@ class BenchmarkEvidenceTest < Minitest::Test class GitHub attr_reader :requests - def initialize(expired: false, unavailable_attempt: nil) + def initialize(expired: false, unavailable_attempt: nil, empty_logs: false, no_jobs: false) @requests = [] @expired = expired @unavailable_attempt = unavailable_attempt + @empty_logs = empty_logs + @no_jobs = no_jobs end def api(path) @@ -30,9 +32,11 @@ def api(path) def pages(path, key) @requests << path if key == "jobs" + return [] if @no_jobs attempt = path[%r{/attempts/(\d+)/}, 1].to_i [{"id" => 100 + attempt, "run_id" => 42, "run_attempt" => attempt}] else + return [] if @no_jobs [{"id" => 99, "name" => "phase", "expired" => @expired}] end end @@ -42,7 +46,8 @@ def download(path, target) if path.include?("/attempts/#{@unavailable_attempt}/") raise BenchmarkCases::Error, "Logs expired" end - File.binwrite(target, "download from #{path}\n") + contents = @empty_logs && path.end_with?("/logs") ? "PK\x05\x06" + "\x00" * 18 : "download from #{path}\n" + File.binwrite(target, contents) end end @@ -74,6 +79,26 @@ def test_missing_attempt_logs_and_expired_artifacts_remain_partial end end + def test_valid_empty_log_archives_are_retained_for_runs_with_no_jobs + with_export(empty_logs: true, no_jobs: true) do |target, result, _| + assert_equal "complete", result.fetch("state") + path = File.join(target, "attempts/1/logs.zip") + assert BenchmarkEvidence.empty_zip?(path) + BenchmarkEvidence.verify_zip(path) + File.binwrite(path, "PK\x05\x06" + "\x00" * 17 + "\x01") + assert_raises(BenchmarkCases::Error) { BenchmarkEvidence.verify_zip(path) } + end + end + + def test_empty_log_archives_with_jobs_remain_partial + with_export(empty_logs: true) do |target, result, _| + assert_equal "partial", result.fetch("state") + assert_equal "Jobs exist but the log archive has no entries", result.fetch("gaps").first.fetch("reason") + manifest = JSON.parse(File.read(File.join(target, "manifest.json"))) + assert_equal 22, manifest.dig("files", "attempts/1/logs.zip", "bytes") + end + end + def test_detects_changed_bytes_and_refuses_an_existing_export_destination with_export do |target, _, github| File.write(File.join(target, "workflow.yml"), "changed\n") diff --git a/test/benchmark_series_test.rb b/test/benchmark_series_test.rb index 4b3969f2a..728920327 100644 --- a/test/benchmark_series_test.rb +++ b/test/benchmark_series_test.rb @@ -38,21 +38,26 @@ def add(value) BenchmarkSeries.record(@directory, path) end - def evidence(run_id: 1, conclusion: "success", logs: "Build and cleanup completed\n", repository: BenchmarkCases::REPOSITORY) + def evidence(run_id: 1, conclusion: "success", logs: "Build and cleanup completed\n", repository: BenchmarkCases::REPOSITORY, no_jobs: false) directory = File.join(@root, "evidence-#{run_id}") FileUtils.mkdir_p(File.join(directory, "attempts", "1")) run = {"id" => run_id, "head_sha" => "a" * 40, "run_attempt" => 1, "status" => "completed", "conclusion" => conclusion} + jobs = no_jobs ? [] : [{"id" => 10, "name" => "Build", "conclusion" => conclusion}] manifest = {"repository" => repository, "run_id" => run_id, "run_url" => "https://github.com/#{repository}/actions/runs/#{run_id}", - "source_sha" => "a" * 40, "state" => "complete", "gaps" => [], "attempts" => [{"attempt" => 1, "job_ids" => [10]}], "artifacts" => [], "files" => {}} + "source_sha" => "a" * 40, "state" => "complete", "gaps" => [], "attempts" => [{"attempt" => 1, "job_ids" => jobs.map { |job| job.fetch("id") }}], "artifacts" => [], "files" => {}} BenchmarkEvidence.retain_json(directory, "run.json", run, manifest) BenchmarkEvidence.retain_json(directory, "attempts/1/run.json", run, manifest) - BenchmarkEvidence.retain_json(directory, "attempts/1/jobs.json", {"total_count" => 1, "jobs" => [{"id" => 10, "name" => "Build", "conclusion" => conclusion}]}, manifest) + BenchmarkEvidence.retain_json(directory, "attempts/1/jobs.json", {"total_count" => jobs.length, "jobs" => jobs}, manifest) BenchmarkEvidence.retain_json(directory, "artifacts.json", {"total_count" => 0, "artifacts" => []}, manifest) BenchmarkEvidence.retain_json(directory, "commit.json", {"sha" => "a" * 40}, manifest) File.write(File.join(directory, "workflow.yml"), "name: Build\n") - File.write(File.join(@root, "log.txt"), logs) - _stdout, stderr, status = Open3.capture3("zip", "-q", File.join(directory, "attempts/1/logs.zip"), "log.txt", chdir: @root) - raise stderr unless status.success? + if no_jobs + File.binwrite(File.join(directory, "attempts/1/logs.zip"), "PK\x05\x06" + "\x00" * 18) + else + File.write(File.join(@root, "log.txt"), logs) + _stdout, stderr, status = Open3.capture3("zip", "-q", File.join(directory, "attempts/1/logs.zip"), "log.txt", chdir: @root) + raise stderr unless status.success? + end %w[workflow.yml attempts/1/logs.zip].each do |relative| path = File.join(directory, relative) manifest["files"][relative] = {"bytes" => File.size(path), "sha256" => Digest::SHA256.file(path).hexdigest} @@ -257,4 +262,15 @@ def test_failed_runs_and_foreign_or_changed_exports_are_not_qualified File.write(path, "changed\n") assert_raises(BenchmarkCases::Error) { BenchmarkSeries.finish(@directory, evidence_directory: File.join(@root, "evidence-1"), sample: 1) } end + + def test_cancellation_before_jobs_start_is_retained_without_measurements + result = BenchmarkSeries.finish(@directory, evidence_directory: evidence(conclusion: "cancelled", no_jobs: true), sample: 1) + assert_equal "cancelled", result["github_conclusion"] + assert_equal "failed", result["status"] + assert_includes result["errors"], "Workflow concluded cancelled" + report = BenchmarkSeries.report(@directory) + assert_empty report["records"] + refute report["valid_for_comparison"] + assert_equal 1, report["failed_completions"].length + end end diff --git a/test/native_case_test.rb b/test/native_case_test.rb index a2c0f8e72..f3d0b1fba 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -18,7 +18,7 @@ def test_fresh_and_rolling_share_the_recipe_but_preserve_publication_behavior fresh = NativeCase.write_action(item, directory: directory, lane: "fresh") rolling = NativeCase.write_action(item, directory: directory, lane: "rolling") assert_equal fresh.dig("runs", "steps", 0, "uses"), rolling.dig("runs", "steps", 0, "uses") - refute fresh.dig("runs", "steps", 0, "with").key?("push_image") + assert_equal "true", fresh.dig("runs", "steps", 0, "with", "load_image") assert_equal "true", rolling.dig("runs", "steps", 0, "with", "push_image") assert_equal "${{ inputs.cli_version }}", fresh.dig("runs", "steps", 0, "with", "cli_version") end @@ -114,6 +114,52 @@ def test_provider_flags_preserve_the_comparator_and_cannot_change_phase_identity end end + def test_native_planning_rejects_missing_output_verification_before_dispatch + %w[chroma duckgres hugo linkerd2].each do |id| + item = BenchmarkCases.load_case(id) + assert_equal "native-fresh-benchmark.yml", BenchmarkCases.plan(item).fetch("workflow") + error = assert_raises(NativeCase::Error) { BenchmarkCases.plan(item, lane: "rolling") } + assert_includes error.message, "requires load_image=true" + end + with_payload("hugo-go") do |item, directory| + path = File.join(directory, item.dig("execution", "native", "action"), "action.yml") + File.write(path, File.read(path).gsub("--verified-output", "")) + error = assert_raises(NativeCase::Error) { NativeCase.verify_report(item.dig("execution", "native"), payload: directory, lane: "fresh") } + assert_includes error.message, "does not report verified output" + end + %w[hugo-go qdrant spring-ai storybook opentelemetry-java].each do |id| + assert_equal "native-fresh-benchmark.yml", BenchmarkCases.plan(BenchmarkCases.load_case(id)).fetch("workflow") + end + end + + def test_native_preparation_rejects_unverified_output_before_creating_a_checkout + Dir.mktmpdir("native-preflight-") do |directory| + target = File.join(directory, "workload") + error = assert_raises(NativeCase::Error) do + item = BenchmarkCases.load_case("n8n") + item["execution"]["native"]["variants"]["docker"]["fresh_inputs"]["load_image"] = "false" + BenchmarkCases.prepare(item, directory: target, native_lane: "fresh", variant: "docker") + end + assert_includes error.message, "requires load_image=true" + refute Dir.exist?(target) + end + end + + def test_native_docker_variants_use_the_shared_loaded_image_check + {"mastodon" => %w[server server-sccache streaming], "posthog" => %w[layers combined], "immich" => [nil]}.each do |id, variants| + variants.each do |variant| + item = BenchmarkCases.load_case(id) + assert_equal "native-fresh-benchmark.yml", BenchmarkCases.plan(item, variant: variant).fetch("workflow") + recipe = NativeCase.resolve(item.dig("execution", "native"), variant) + assert_equal "true", recipe.dig("fresh_inputs", "load_image") + action = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, "cases", id, "payload", recipe.fetch("action"), "action.yml"))) + steps = action.dig("runs", "steps") + assert_operator steps.index { |step| step["id"] == "build_timing" }, :<, steps.index { |step| step["id"] == "output_verification" } + assert_operator steps.index { |step| step["id"] == "output_verification" }, :<, steps.index { |step| step["name"] == "Write the benchmark phase evidence" } + end + end + end + def test_unreviewed_recipe_input_and_unsafe_suffix_are_rejected with_payload("hugo-go") do |item, directory| assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh", suffix: "\nOTHER=value") } From 5886cff1d2bebbd7129660124ea6d961b365c546 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 08:04:34 +0100 Subject: [PATCH 05/17] Keep methodology issues in generated series reports --- docs/process.md | 6 ++++++ scripts/benchmark-series.rb | 22 ++++++++++++++++++++- test/benchmark_series_test.rb | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/process.md b/docs/process.md index f4edb3107..253ed45ae 100644 --- a/docs/process.md +++ b/docs/process.md @@ -210,6 +210,12 @@ uses the declared provider byte measurement for its primary median and range. Aggregate reporting retains suspected runner-variance samples and attaches its diagnostic evidence. A variance flag does not establish the cause of a slow or fast run and must not remove its timing from the comparison. +When review finds a measurement or fairness issue, keep the original plan and +records. Add `methodology-review.json` beside the series plan with +`schema_version: 1`, the exact `plan_sha256`, nonempty `issues`, and HTTPS +`evidence_links`. Reporting and the catalog retain all observations and mark the +series ineligible for a comparative claim. A corrected definition needs a new +series before execution; an old successful run must not acquire a revised scope. Provider-reported bytes describe the selected cache, not total workspace usage or billable storage. The BoringCache probe uses exact resolved tags and its reported KV or archive size; the GitHub probe uses the selected cache key's diff --git a/scripts/benchmark-series.rb b/scripts/benchmark-series.rb index 7744e2429..7d94f1a00 100644 --- a/scripts/benchmark-series.rb +++ b/scripts/benchmark-series.rb @@ -188,7 +188,7 @@ def self.report(directory, write: true) missing_completions = run_ids - completions.map { |completion| completion.fetch("run_id") } failed_completions = completions.reject { |completion| completion["status"] == "success" } execution_verified = !run_ids.empty? && !run_ids.include?("") && missing_completions.empty? && failed_completions.empty? - methodology_issues = [] + methodology_issues = methodology_review(directory, plan) if plan.fetch("lane") == "rolling" methodology_issues << "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic." end @@ -256,6 +256,26 @@ def self.report(directory, write: true) result end + def self.methodology_review(directory, plan) + path = File.join(directory, "methodology-review.json") + return [] unless File.file?(path) + + review = JSON.parse(File.read(path)) + unless review.is_a?(Hash) && review.keys.sort == %w[evidence_links issues plan_sha256 schema_version] && review["schema_version"] == 1 + raise Error, "Invalid methodology review contract" + end + raise Error, "Methodology review belongs to another plan" unless review["plan_sha256"] == plan.fetch("plan_sha256") + issues = review["issues"] + unless issues.is_a?(Array) && !issues.empty? && issues.all? { |issue| issue.is_a?(String) && !issue.strip.empty? } + raise Error, "Methodology review requires explicit issues" + end + links = review["evidence_links"] + unless links.is_a?(Array) && !links.empty? && links.all? { |link| link.is_a?(String) && link.match?(%r{\Ahttps://[^\s]+\z}) } + raise Error, "Methodology review requires evidence links" + end + issues.uniq + end + def self.statistics(values) return nil if values.empty? sorted = values.sort diff --git a/test/benchmark_series_test.rb b/test/benchmark_series_test.rb index 728920327..5358bf8b3 100644 --- a/test/benchmark_series_test.rb +++ b/test/benchmark_series_test.rb @@ -114,6 +114,42 @@ def test_complete_correctness_proof_does_not_qualify_a_provider_comparison refute_includes File.read(File.join(@directory, "report.md")), "Median correctness" end + def test_methodology_review_keeps_complete_observations_but_blocks_comparative_claims + @directory = File.join(@root, "reviewed") + plan = BenchmarkSeries.create(@case, directory: @directory, series: "screening", lane: "fresh", variant: "release") + [1, 2].each do |sample| + %w[cold warm].each do |phase| + %w[boringcache actions-cache].each { |provider| add(record(sample: sample, provider: provider).merge("phase" => phase, "lane" => "fresh")) } + end + BenchmarkSeries.finish(@directory, evidence_directory: evidence(run_id: sample), sample: sample) + end + assert BenchmarkSeries.report(@directory)["valid_for_comparison"] + review = {"schema_version" => 1, "plan_sha256" => plan.fetch("plan_sha256"), + "issues" => ["Setup timing includes dependency installation"], "evidence_links" => ["https://example.org/evidence"]} + File.write(File.join(@directory, "methodology-review.json"), JSON.generate(review)) + result = BenchmarkSeries.report(@directory) + assert result["complete"] + assert result["execution_verified"] + refute result["valid_for_comparison"] + assert_equal 8, result["records"].length + assert_equal [], result["exclusions"] + assert_equal review["issues"], result["methodology_issues"] + assert_includes File.read(File.join(@directory, "report.md")), review["issues"].first + end + + def test_rejects_methodology_reviews_for_another_plan_or_without_evidence + plan = BenchmarkSeries.load(@directory) + review = {"schema_version" => 1, "plan_sha256" => "0" * 64, + "issues" => ["Timing issue"], "evidence_links" => ["https://example.org/evidence"]} + path = File.join(@directory, "methodology-review.json") + File.write(path, JSON.generate(review)) + assert_raises(BenchmarkSeries::Error) { BenchmarkSeries.report(@directory) } + review["plan_sha256"] = plan.fetch("plan_sha256") + review["evidence_links"] = [] + File.write(path, JSON.generate(review)) + assert_raises(BenchmarkSeries::Error) { BenchmarkSeries.report(@directory) } + end + def test_storage_comparison_uses_provider_bytes_for_the_primary_measurement @case["comparison"]["primary_metric"] = "storage_bytes" @directory = File.join(@root, "storage") From b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 08:08:53 +0100 Subject: [PATCH 06/17] Exclude Storybook preparation and preserve qualification requests --- cases/storybook/case.json | 2 +- .../actions/storybook-nx-benchmark/action.yml | 19 +- data/latest/series.json | 522 +++++++- docs/migration.md | 4 + migration/evidence.json | 16 + ...image-output-qualification-2026-10-03.json | 142 +++ .../1-native-fresh-benchmark.yml.json | 34 + .../series.json | 56 + .../series.json | 56 + .../1-native-fresh-benchmark.yml.json | 34 + .../1-native-fresh-benchmark.yml.json | 34 + .../1-native-fresh-benchmark.yml.json | 34 + .../1-native-fresh-benchmark.yml.json | 34 + .../1-native-fresh-benchmark.yml.json | 39 + .../1-native-fresh-benchmark.yml.json | 40 + .../completions/37087350856.json | 13 + .../shared-turbo-qualification-01/evidence.md | 15 + .../interpretation.md | 20 + .../shared-turbo-qualification-01/report.json | 1055 +++++++++++++++++ .../shared-turbo-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 219 ++++ .../runs/1-actions-cache-warm.json | 229 ++++ .../runs/1-boringcache-cold.json | 256 ++++ .../runs/1-boringcache-warm.json | 256 ++++ .../1-native-fresh-benchmark.yml.json | 38 + .../build-reuse-qualification-01/series.json | 44 + test/native_case_test.rb | 21 +- 27 files changed, 3184 insertions(+), 68 deletions(-) create mode 100644 migration/image-output-qualification-2026-10-03.json create mode 100644 results/chroma/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/series.json create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/series.json create mode 100644 results/duckgres/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/hugo/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/immich/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/linkerd2/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/mastodon/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/n8n/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/n8n/shared-turbo-qualification-01/completions/37087350856.json create mode 100644 results/n8n/shared-turbo-qualification-01/evidence.md create mode 100644 results/n8n/shared-turbo-qualification-01/interpretation.md create mode 100644 results/n8n/shared-turbo-qualification-01/report.json create mode 100644 results/n8n/shared-turbo-qualification-01/report.md create mode 100644 results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/n8n/shared-turbo-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/n8n/shared-turbo-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/posthog/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/storybook/build-reuse-qualification-01/series.json diff --git a/cases/storybook/case.json b/cases/storybook/case.json index 47251ad21..740cacbc4 100644 --- a/cases/storybook/case.json +++ b/cases/storybook/case.json @@ -71,7 +71,7 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", diff --git a/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml b/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml index bec0349ae..ddd1e112a 100644 --- a/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml +++ b/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml @@ -122,6 +122,15 @@ runs: working-directory: "." cache-profiles: storybook-nx fail-on-cache-miss: true + - name: Capture cache setup timing + id: setup_timing + shell: bash + env: + SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" + run: | + set -euo pipefail + now="$(date +%s)" + echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - name: Install Storybook dependencies shell: bash working-directory: upstream @@ -134,13 +143,7 @@ runs: - name: Start the build timer id: build_timer shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Build the Storybook benchmark sandbox shell: bash run: ruby ./scripts/run-benchmark-plan.rb nx --working-directory upstream @@ -176,7 +179,7 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" + SETUP_SECONDS: "${{ steps.setup_timing.outputs.setup_seconds }}" BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" CACHE_HIT: "${{ steps.boringcache-publish.outputs.cache-hit || steps.boringcache-restore.outputs.cache-hit || steps.actions_cache_0.outputs.cache-hit || steps.actions_cache_1.outputs.cache-hit diff --git a/data/latest/series.json b/data/latest/series.json index 9f24d91e5..14dbe47da 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -34,7 +34,7 @@ }, "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7", "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -62,13 +62,33 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "chroma", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:28Z", + "run_id": 37104573058, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104573058" + ] }, { "case_id": "deno", @@ -526,6 +546,65 @@ ], "runs_without_archives": [] }, + { + "case_id": "deno", + "series_id": "rolling-cargo-product-seed-advance-01", + "plan_path": "results/deno/rolling-cargo-product-seed-advance-01/series.json", + "lane": "rolling", + "variant": "cargo-product", + "sample_count": 1, + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "plan_sha256": "1fcb4e930c7efb0c363b76bb4a92c3ed26305ce8ddf087c465aa0481d009fbe9", + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "deno", "series_id": "rolling-compiler-only-qualification", @@ -641,6 +720,65 @@ ], "runs_without_archives": [] }, + { + "case_id": "deno", + "series_id": "rolling-compiler-only-seed-advance-01", + "plan_path": "results/deno/rolling-compiler-only-seed-advance-01/series.json", + "lane": "rolling", + "variant": "compiler-only", + "sample_count": 1, + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "plan_sha256": "184bb3883ff8ca58307b51e80cb069ecd6f0867f90dd2ead6492a78414585a26", + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "commit", + "boringcache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "deno", "series_id": "wrapper-qualification", @@ -1153,7 +1291,7 @@ }, "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20", "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -1181,13 +1319,33 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "duckgres", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:48Z", + "run_id": 37104593175, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104593175" + ] }, { "case_id": "hugo-go", @@ -1748,7 +1906,7 @@ }, "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700", "definition_sha256": "200bdce61b6cbba1361be02b435abbe9c4750e77dd72058281ba1c1e8a8882f4", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -1776,13 +1934,33 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:50Z", + "run_id": 37104595025, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104595025" + ] }, { "case_id": "immich", @@ -1815,7 +1993,7 @@ }, "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -1843,13 +2021,33 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "immich", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:59Z", + "run_id": 37104602471, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104602471" + ] }, { "case_id": "linkerd2", @@ -1882,7 +2080,7 @@ }, "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -1910,13 +2108,33 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "linkerd2", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:52Z", + "run_id": 37104596700, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104596700" + ] }, { "case_id": "mastodon", @@ -1949,7 +2167,7 @@ }, "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -1977,13 +2195,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "server", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:55Z", + "run_id": 37104599574, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104599574" + ] }, { "case_id": "n8n", @@ -2016,7 +2255,7 @@ }, "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -2044,13 +2283,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "docker", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:54Z", + "run_id": 37104598178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37104598178" + ] }, { "case_id": "n8n", @@ -2083,34 +2343,74 @@ }, "plan_sha256": "035df772551717a7a2d369d39994c744f270815a5481833e7a494a06097dcae9", "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 125, + "min": 125, + "max": 125 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 4, + "min": 4, + "max": 4 + }, + "storage": { + "median": 52937490, + "min": 52937490, + "max": 52937490 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 132, + "min": 132, + "max": 132 + }, + "storage": { + "median": 46409937, + "min": 46409937, + "max": 46409937 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 9, + "min": 9, + "max": 9 + }, + "storage": { + "median": 51597365, + "min": 51597365, + "max": 51597365 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "n8n", @@ -2131,14 +2431,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37087350856" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", + "sample": 1, + "run_id": "37087350856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856", + "source_sha": "6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "25fcd514b9aa761dbc44baa4a988e563bf43e071a2538617921e2c23a7e5c5b0" + } + ], + "report_state": "current", + "report_path": "results/n8n/shared-turbo-qualification-01/report.json", + "interpretation_path": "results/n8n/shared-turbo-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37087350856, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-n8n-turbo-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-n8n-turbo-2026-10-03/benchmarks-37087350856.tar", + "bundle_bytes": 484864, + "sha256": "8ebeab6ffdf273e07caa14b311620c1889a43ba722f2b4ce64e592237835e383", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "obs-studio", @@ -2732,6 +3058,94 @@ }, "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:57Z", + "run_id": 37104600856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + } + ], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [ + "37104600856" + ] + }, + { + "case_id": "storybook", + "series_id": "build-reuse-qualification-01", + "plan_path": "results/storybook/build-reuse-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "d463583c8fb964e5c3721430687e155f6579a1a423afa5508f758069e22942b5", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, diff --git a/docs/migration.md b/docs/migration.md index bf450c2f0..ba1c99f89 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -57,6 +57,10 @@ timings must not be combined with that earlier projection. n8n Turbo's primary measurement now captures cache setup and the build separately from its dependency installation. The install still runs before the build. Earlier results retain their original timed scope and must not be mixed with this series. +Storybook's first fresh screen also timed dependency installation and sandbox +creation in its setup metric. Its plan-bound methodology review prevents a +comparative claim while retaining every original observation. The replacement +recipe stops the setup timer before those operations and needs a new series. GitHub's default CodeQL configuration now scans Ruby and Actions. Python was removed from its language list because this branch removes the maintained Python reporter; query suite, threat model, weekly schedule, and runner settings were diff --git a/migration/evidence.json b/migration/evidence.json index 00bf48a8f..972ba1831 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -375,6 +375,22 @@ "execution_qualification": "Passed one-sample combined-cache cold/changed-source correctness and completion checks; no provider comparison is declared", "scope": "Available attempts, jobs, logs, artifacts, commit metadata, and workflow for this run; not an original-repository audit", "repository_retirement": "Keep originals until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37087350856, + "verified_files": 15, + "gaps": [], + "sha256": "8ebeab6ffdf273e07caa14b311620c1889a43ba722f2b4ce64e592237835e383", + "bundle_bytes": 484864, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-n8n-turbo-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-n8n-turbo-2026-10-03/benchmarks-37087350856.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Passed scoped output and completion checks; one sample; publication and measurement interpretation remain unreviewed", + "repository_retirement": "Original repositories remain until all migration gates pass" } ], "bundles": [ diff --git a/migration/image-output-qualification-2026-10-03.json b/migration/image-output-qualification-2026-10-03.json new file mode 100644 index 000000000..659f89768 --- /dev/null +++ b/migration/image-output-qualification-2026-10-03.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "scope": "One-sample loaded-image qualification; cache API snapshots are context, not per-phase occupancy or evidence that eviction did not occur", + "publication": "unreviewed", + "cache_observations": { + "usage": { + "endpoint": "repos/boringcache/benchmarks/actions/cache/usage", + "collected_at": "2026-10-03T06:54:17Z", + "response": { + "full_name": "boringcache/benchmarks", + "active_caches_size_in_bytes": 4709831954, + "active_caches_count": 182 + } + }, + "storage-limit": { + "endpoint": "repos/boringcache/benchmarks/actions/cache/storage-limit", + "collected_at": "2026-10-03T06:54:17Z", + "response": { + "max_cache_size_gb": 10 + } + } + }, + "dispatches": [ + { + "case_id": "chroma", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104573058, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058", + "requested_at": "2026-10-03T06:54:28Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "chroma", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "duckgres", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104593175, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175", + "requested_at": "2026-10-03T06:54:48Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "duckgres", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "hugo", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104595025, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "requested_at": "2026-10-03T06:54:50Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "immich", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104602471, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471", + "requested_at": "2026-10-03T06:54:59Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "immich", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "linkerd2", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104596700, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700", + "requested_at": "2026-10-03T06:54:52Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "linkerd2", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "mastodon", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104599574, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574", + "requested_at": "2026-10-03T06:54:55Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "server", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "n8n", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104598178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178", + "requested_at": "2026-10-03T06:54:54Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "docker", + "series_id": "image-output-qualification-01", + "sample": "1" + } + }, + { + "case_id": "posthog", + "workflow": "native-fresh-benchmark.yml", + "run_id": 37104600856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", + "requested_at": "2026-10-03T06:54:57Z", + "ref": "image-output-qualification-2026-10-03", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "image-output-qualification-01", + "sample": "1" + } + } + ] +} diff --git a/results/chroma/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/chroma/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..dbe19cb83 --- /dev/null +++ b/results/chroma/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "chroma", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "chroma", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T06:54:28Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104573058, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058" +} diff --git a/results/deno/rolling-cargo-product-seed-advance-01/series.json b/results/deno/rolling-cargo-product-seed-advance-01/series.json new file mode 100644 index 000000000..ac128f377 --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/series.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-cargo-product-seed-advance-01", + "created_at": "2026-10-03T07:08:53Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "variant": "cargo-product", + "workflow_inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085348479-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "cargo-product" + }, + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "verification": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ], + "plan_sha256": "1fcb4e930c7efb0c363b76bb4a92c3ed26305ce8ddf087c465aa0481d009fbe9" +} diff --git a/results/deno/rolling-compiler-only-seed-advance-01/series.json b/results/deno/rolling-compiler-only-seed-advance-01/series.json new file mode 100644 index 000000000..8c04d018f --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/series.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-compiler-only-seed-advance-01", + "created_at": "2026-10-03T07:08:53Z", + "lane": "rolling", + "phases": [ + "commit" + ], + "sample_count": 1, + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "variant": "compiler-only", + "workflow_inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085349929-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "compiler-only" + }, + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "verification": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ], + "plan_sha256": "184bb3883ff8ca58307b51e80cb069ecd6f0867f90dd2ead6492a78414585a26" +} diff --git a/results/duckgres/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/duckgres/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..b6a432e7f --- /dev/null +++ b/results/duckgres/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "duckgres", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "duckgres", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T06:54:48Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104593175, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175" +} diff --git a/results/hugo/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/hugo/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..c04e31d02 --- /dev/null +++ b/results/hugo/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T06:54:50Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104595025, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025" +} diff --git a/results/immich/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/immich/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..c09f1a228 --- /dev/null +++ b/results/immich/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "immich", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "immich", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T06:54:59Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104602471, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471" +} diff --git a/results/linkerd2/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/linkerd2/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..3fd2a15e4 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "linkerd2", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "linkerd2", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T06:54:52Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104596700, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700" +} diff --git a/results/mastodon/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/mastodon/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..cbbf964f4 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "server", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-03T06:54:55Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104599574, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574" +} diff --git a/results/n8n/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/n8n/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..754330113 --- /dev/null +++ b/results/n8n/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "docker", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-03T06:54:54Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104598178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178" +} diff --git a/results/n8n/shared-turbo-qualification-01/completions/37087350856.json b/results/n8n/shared-turbo-qualification-01/completions/37087350856.json new file mode 100644 index 000000000..9cb3fff72 --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/completions/37087350856.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", + "sample": 1, + "run_id": "37087350856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856", + "source_sha": "6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "25fcd514b9aa761dbc44baa4a988e563bf43e071a2538617921e2c23a7e5c5b0" +} diff --git a/results/n8n/shared-turbo-qualification-01/evidence.md b/results/n8n/shared-turbo-qualification-01/evidence.md new file mode 100644 index 000000000..25296a4a6 --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/evidence.md @@ -0,0 +1,15 @@ +# Preserved qualification evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-n8n-turbo-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37087350856 | 484864 | `8ebeab6ffdf273e07caa14b311620c1889a43ba722f2b4ce64e592237835e383` | 15 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +The scope covers available attempts, jobs, logs, artifacts, commit metadata, +and workflow for this run. It does not audit the whole original repository. + +The [interpretation](interpretation.md) records execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/n8n/shared-turbo-qualification-01/interpretation.md b/results/n8n/shared-turbo-qualification-01/interpretation.md new file mode 100644 index 000000000..e7995337d --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/interpretation.md @@ -0,0 +1,20 @@ +# n8n Turbo fresh screening + +[Run 37087350856](https://github.com/boringcache/benchmarks/actions/runs/37087350856) +completed the predeclared cold and identical-source replay for both providers from +signed harness `6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c`, using CLI 1.33.0. +The built CLI entrypoint, matched source and runner environment, job completion, +and post-step checks passed. Original observations remain in the +[generated report](report.json). + +Build plus measured cache setup was 132 seconds for BoringCache and 125 for +Actions Cache cold, then 9 and 4 seconds respectively for replay. Both slower +BoringCache observations remain included. Dependency installation runs outside +the setup and build timers. The output check establishes the built entrypoint; +it does not test application runtime behavior. Queue and job duration are context. + +Actions Cache cold storage is unmeasured. Selected provider byte counts do not +establish physical storage efficiency. There is one sample per phase and provider. +The [durable evidence](evidence.md) was independently downloaded and verified. +Publication remains unreviewed. Docker variants, changed-source reuse, rolling +publication, and caller cutover require separate qualification. diff --git a/results/n8n/shared-turbo-qualification-01/report.json b/results/n8n/shared-turbo-qualification-01/report.json new file mode 100644 index 000000000..8b7308b0e --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/report.json @@ -0,0 +1,1055 @@ +{ + "schema_version": 1, + "plan_sha256": "035df772551717a7a2d369d39994c744f270815a5481833e7a494a06097dcae9", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", + "sample": 1, + "run_id": "37087350856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856", + "source_sha": "6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "25fcd514b9aa761dbc44baa4a988e563bf43e071a2538617921e2c23a7e5c5b0" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 125, + "min": 125, + "max": 125 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 4, + "min": 4, + "max": 4 + }, + "storage": { + "median": 52937490, + "min": 52937490, + "max": 52937490 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 132, + "min": 132, + "max": 132 + }, + "storage": { + "median": 46409937, + "min": 46409937, + "max": 46409937 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 9, + "min": 9, + "max": 9 + }, + "storage": { + "median": 51597365, + "min": 51597365, + "max": 51597365 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 125, + "workflow_seconds": 167, + "total_seconds": 125, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 125 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169954", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 2, + "workflow_seconds": 51, + "total_seconds": 4, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 4 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 52937490, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "turbo-n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": true, + "measured_bytes": 52937490, + "total_bytes": 52937490, + "observations": [ + { + "id": 8443866757, + "key": "turbo-n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "ref": "refs/heads/refs/tags/native-workload-qualification-2026-10-03", + "version": "7c311f787638ef390f5a96a444bb6b72bc00284a34a366bdacae6430cb809a9a", + "created_at": "2026-10-03T03:00:04.683757Z", + "last_accessed_at": "2026-10-03T03:00:54.737161Z", + "size_in_bytes": 52937490 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169960", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 127, + "workflow_seconds": 174, + "total_seconds": 132, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 132 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46409937, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 46409937, + "unmeasured_tags": [], + "total_bytes": 46409937, + "observations": [ + { + "requested_tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46409937, + "compressed_size": 46409937, + "size": 46409937 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169955", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 4, + "build_seconds": 5, + "workflow_seconds": 50, + "total_seconds": 9, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 9 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 51597365, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 51597365, + "unmeasured_tags": [], + "total_bytes": 51597365, + "observations": [ + { + "requested_tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 51597365, + "compressed_size": 51597365, + "size": 51597365 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169961", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/shared-turbo-qualification-01/report.md b/results/n8n/shared-turbo-qualification-01/report.md new file mode 100644 index 000000000..72e69c1b6 --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/report.md @@ -0,0 +1,20 @@ +# n8n: shared-turbo-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 125 | 125–125 | unmeasured | 0 | +| warm | actions-cache | 1 | 4 | 4–4 | 52937490 | 1 | +| cold | boringcache | 1 | 132 | 132–132 | 46409937 | 1 | +| warm | boringcache | 1 | 9 | 9–9 | 51597365 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-cold.json b/results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..cff9c0dac --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,219 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 125, + "workflow_seconds": 167, + "total_seconds": 125, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 125 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "turbo-n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169954", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] +} diff --git a/results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-warm.json b/results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..aa983d5c8 --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,229 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 2, + "workflow_seconds": 51, + "total_seconds": 4, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 4 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 52937490, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "turbo-n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "complete": true, + "measured_bytes": 52937490, + "total_bytes": 52937490, + "observations": [ + { + "id": 8443866757, + "key": "turbo-n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "ref": "refs/heads/refs/tags/native-workload-qualification-2026-10-03", + "version": "7c311f787638ef390f5a96a444bb6b72bc00284a34a366bdacae6430cb809a9a", + "created_at": "2026-10-03T03:00:04.683757Z", + "last_accessed_at": "2026-10-03T03:00:54.737161Z", + "size_in_bytes": 52937490 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169960", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] +} diff --git a/results/n8n/shared-turbo-qualification-01/runs/1-boringcache-cold.json b/results/n8n/shared-turbo-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..09324d3d9 --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,256 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 127, + "workflow_seconds": 174, + "total_seconds": 132, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 132 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46409937, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 46409937, + "unmeasured_tags": [], + "total_bytes": 46409937, + "observations": [ + { + "requested_tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46409937, + "compressed_size": 46409937, + "size": 46409937 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169955", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] +} diff --git a/results/n8n/shared-turbo-qualification-01/runs/1-boringcache-warm.json b/results/n8n/shared-turbo-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..ad9dbc6da --- /dev/null +++ b/results/n8n/shared-turbo-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,256 @@ +{ + "schema_version": 1, + "benchmark": "n8n", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "turbo", + "mode": "turbo", + "adapter": "turborepo", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 4, + "build_seconds": 5, + "workflow_seconds": 50, + "total_seconds": 9, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 9 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo", + "workspace": "boringcache/benchmarks", + "storage_bytes": 51597365, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 51597365, + "unmeasured_tags": [], + "total_bytes": 51597365, + "observations": [ + { + "requested_tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "tag": "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 51597365, + "compressed_size": 51597365, + "size": 51597365 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "turbo", + "resolved_tags": [ + "n8n-series-shared-turbo-qualification-01-s1-r37087350856-a1-turbo-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37087350856", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "native-workload-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000169961", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37087350856-1", + "series": { + "id": "shared-turbo-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37087350856" + ] +} diff --git a/results/posthog/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..23d9ddf10 --- /dev/null +++ b/results/posthog/image-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-03T06:54:57Z", + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "run_id": 37104600856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856" +} diff --git a/results/storybook/build-reuse-qualification-01/series.json b/results/storybook/build-reuse-qualification-01/series.json new file mode 100644 index 000000000..213e18c62 --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/series.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "case_id": "storybook", + "series_id": "build-reuse-qualification-01", + "created_at": "2026-10-03T07:05:46Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "variant": null, + "workflow_inputs": { + "case_id": "storybook", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "plan_sha256": "d463583c8fb964e5c3721430687e155f6579a1a423afa5508f758069e22942b5" +} diff --git a/test/native_case_test.rb b/test/native_case_test.rb index f3d0b1fba..0a5b8aad0 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -72,15 +72,26 @@ def test_native_variant_recipes_are_validated_before_selection end def test_n8n_dependency_installation_does_not_increase_build_and_reuse_time - path = File.join(BenchmarkCases::ROOT, "cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml") + assert_dependency_setup_excluded("n8n", "n8n-turbo-benchmark", ["Install n8n dependencies"], "Build n8n") + end + + def test_storybook_installation_and_sandbox_creation_do_not_increase_build_and_reuse_time + assert_dependency_setup_excluded("storybook", "storybook-nx-benchmark", ["Install Storybook dependencies", "Create the upstream Storybook sandbox"], "Build the Storybook benchmark sandbox") + end + + def assert_dependency_setup_excluded(case_id, action, setup_names, build_name) + path = File.join(BenchmarkCases::ROOT, "cases", case_id, "payload/.github/actions", action, "action.yml") steps = YAML.safe_load(File.read(path)).dig("runs", "steps") - install = steps.index { |step| step["name"] == "Install n8n dependencies" } timer = steps.index { |step| step["id"] == "build_timer" } - assert_operator install, :<, timer - assert_operator timer, :<, steps.index { |step| step["name"] == "Build n8n" } + setup_names.each do |name| + setup = steps.index { |step| step["name"] == name } + assert_operator steps.index { |step| step["id"] == "setup_timing" }, :<, setup + assert_operator setup, :<, timer + end + assert_operator timer, :<, steps.index { |step| step["name"] == build_name } reporter = steps.find { |step| step["name"] == "Write the benchmark phase evidence" } assert_equal "${{ steps.setup_timing.outputs.setup_seconds }}", reporter.dig("env", "SETUP_SECONDS") - Dir.mktmpdir("n8n-timing-") do |directory| + Dir.mktmpdir("benchmark-timing-") do |directory| clock = File.join(directory, "clock") output = File.join(directory, "output") date = File.join(directory, "date") From cd7c9b520c2d4b6fb846941c915bf058e4608272 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 08:11:10 +0100 Subject: [PATCH 07/17] Record Storybook and Deno seed advancement dispatches --- data/latest/series.json | 84 +++++++++++++++++-- .../1-deno-deno-cargo-rolling-chain.yml.json | 44 ++++++++++ .../1-deno-deno-cargo-rolling-chain.yml.json | 44 ++++++++++ .../1-native-fresh-benchmark.yml.json | 34 ++++++++ 4 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json create mode 100644 results/storybook/build-reuse-qualification-01/dispatches/1-native-fresh-benchmark.yml.json diff --git a/data/latest/series.json b/data/latest/series.json index 14dbe47da..d38700236 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -581,7 +581,7 @@ }, "plan_sha256": "1fcb4e930c7efb0c363b76bb4a92c3ed26305ce8ddf087c465aa0481d009fbe9", "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -597,13 +597,36 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "deno", + "repository": "boringcache/benchmarks", + "workflow": "deno-deno-cargo-rolling-chain.yml", + "lane": "rolling", + "inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085348479-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "cargo-product", + "series_id": "rolling-cargo-product-seed-advance-01", + "sample": "1" + }, + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T07:10:39Z", + "run_id": 37105483797, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37105483797" + ] }, { "case_id": "deno", @@ -755,7 +778,7 @@ }, "plan_sha256": "184bb3883ff8ca58307b51e80cb069ecd6f0867f90dd2ead6492a78414585a26", "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -771,13 +794,36 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "deno", + "repository": "boringcache/benchmarks", + "workflow": "deno-deno-cargo-rolling-chain.yml", + "lane": "rolling", + "inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085349929-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "compiler-only", + "series_id": "rolling-compiler-only-seed-advance-01", + "sample": "1" + }, + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T07:10:41Z", + "run_id": 37105485279, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37105485279" + ] }, { "case_id": "deno", @@ -3146,7 +3192,7 @@ }, "plan_sha256": "d463583c8fb964e5c3721430687e155f6579a1a423afa5508f758069e22942b5", "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -3174,13 +3220,33 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "storybook", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook", + "cli_version": "v1.33.0", + "series_id": "build-reuse-qualification-01", + "sample": "1" + }, + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T07:10:42Z", + "run_id": 37105487087, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37105487087" + ] }, { "case_id": "zed", diff --git a/results/deno/rolling-cargo-product-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json b/results/deno/rolling-cargo-product-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json new file mode 100644 index 000000000..54a309d65 --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json @@ -0,0 +1,44 @@ +{ + "case_id": "deno", + "repository": "boringcache/benchmarks", + "workflow": "deno-deno-cargo-rolling-chain.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085348479-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "cargo-product", + "series_id": "rolling-cargo-product-seed-advance-01", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ], + "schema_version": 1, + "requested_at": "2026-10-03T07:10:39Z", + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "run_id": 37105483797, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797" +} diff --git a/results/deno/rolling-compiler-only-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json b/results/deno/rolling-compiler-only-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json new file mode 100644 index 000000000..3333a084c --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/dispatches/1-deno-deno-cargo-rolling-chain.yml.json @@ -0,0 +1,44 @@ +{ + "case_id": "deno", + "repository": "boringcache/benchmarks", + "workflow": "deno-deno-cargo-rolling-chain.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085349929-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "compiler-only", + "series_id": "rolling-compiler-only-seed-advance-01", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ], + "schema_version": 1, + "requested_at": "2026-10-03T07:10:41Z", + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "run_id": 37105485279, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279" +} diff --git a/results/storybook/build-reuse-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/storybook/build-reuse-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..d35912858 --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "storybook", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "storybook", + "cli_version": "v1.33.0", + "series_id": "build-reuse-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T07:10:42Z", + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "run_id": 37105487087, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087" +} From 91433e6ac97a0724db0740043f790d37ad1bcc5f Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 08:26:19 +0100 Subject: [PATCH 08/17] Select reviewed workloads in canary requests and receipts --- docs/nightly-canaries.md | 7 +++ scripts/check-registry-alignment.rb | 23 ++++++++++ scripts/nightly-canaries.rb | 20 ++++++-- suites/published.json | 71 +++++++++++++++++++++-------- test/nightly_canaries_test.rb | 20 ++++++++ test/publish_index_test.rb | 16 +++++++ 6 files changed, 136 insertions(+), 21 deletions(-) diff --git a/docs/nightly-canaries.md b/docs/nightly-canaries.md index 4bf594e59..0c954db9b 100644 --- a/docs/nightly-canaries.md +++ b/docs/nightly-canaries.md @@ -4,6 +4,13 @@ Central `canary.yml` dispatches the active fresh workflows registered in `suites/published.json`. It selects the latest published CLI canary with a complete set of release assets and dispatches its registered fresh workflows with that exact `cli_version`. Archived repositories are excluded. +Entries with several workload variants declare `fresh_inputs` in that registry. +The dispatcher keeps case and variant selectors in each requested run and in +its receipt; distinct variants of a shared workflow remain distinct requests. +Registry checks resolve those inputs through the case planner and reject an +unselected variant or a diagnostic matrix before caller configuration is accepted. +The selected PostHog layer, Mastodon server/streaming, n8n workloads, Zed combined +layer, and Deno Cargo-product paths do not imply coverage of every other variant. Central dispatch is manual until OIDC and workload qualification pass. The original repositories retain their nightly schedules during cutover. Enable the diff --git a/scripts/check-registry-alignment.rb b/scripts/check-registry-alignment.rb index 353014cc7..bfda7a68e 100755 --- a/scripts/check-registry-alignment.rb +++ b/scripts/check-registry-alignment.rb @@ -28,6 +28,29 @@ %w[workflow fresh_workflow].each do |field| errors << "#{id}: unregistered #{row[field]}" if row[field] && !paths.include?(row[field]) end + next unless item && row["fresh_workflow"] + entry = item.dig("execution", "workflows").find { |candidate| File.basename(candidate.fetch("path")) == row.fetch("fresh_workflow") } + next unless entry + if entry.fetch("lane") != "fresh" + errors << "#{id}: fresh callers must select a reviewed fresh workflow, not a diagnostic matrix" + next + end + inputs = row.fetch("fresh_inputs", {}) + unless inputs.is_a?(Hash) && inputs.all? { |name, value| name.is_a?(String) && value.is_a?(String) } && !inputs.key?("cli_version") + errors << "#{id}: fresh selectors must be string inputs without cli_version" + next + end + inputs = {"case_id" => id}.merge(inputs) if row.fetch("fresh_workflow").start_with?("native-") + variant = inputs[entry["variant_input"]] + if entry["variants"] && !entry.fetch("variants").include?(variant) + errors << "#{id}: fresh caller must select an explicit reviewed variant" + next + end + begin + BenchmarkCases.plan(item, lane: "fresh", workflow: row.fetch("fresh_workflow"), inputs: inputs, variant: variant) + rescue BenchmarkCases::Error, NativeCase::Error => error + errors << "#{id}: #{error.message}" + end end ids = published.map { |row| row.fetch("benchmark") } errors << "Duplicate published benchmark IDs" unless ids.uniq.length == ids.length diff --git a/scripts/nightly-canaries.rb b/scripts/nightly-canaries.rb index d500c56ab..ff330ab10 100755 --- a/scripts/nightly-canaries.rb +++ b/scripts/nightly-canaries.rb @@ -38,7 +38,16 @@ def published_canary(release) def targets(benchmarks) benchmarks.reject { |item| item["archived"] }.map do |item| - inputs = item.fetch("fresh_workflow").start_with?("native-") ? {"case_id" => item.fetch("case_id")} : {} + inputs = item.fetch("fresh_inputs", {}) + unless inputs.is_a?(Hash) && inputs.all? { |key, value| key.is_a?(String) && value.is_a?(String) } && !inputs.key?("cli_version") + raise Error, "Fresh workflow selectors must be string inputs; cli_version belongs to the dispatch" + end + inputs = inputs.dup + if item.fetch("fresh_workflow").start_with?("native-") + id = item.fetch("case_id") + raise Error, "Fresh workflow selector belongs to another case" if inputs.key?("case_id") && inputs["case_id"] != id + inputs["case_id"] = id + end [item.fetch("source_repo"), item.fetch("fresh_workflow"), inputs] end.uniq end @@ -141,7 +150,7 @@ def collect(summary:, benchmarks: BENCHMARKS, now: Time.now.utc) passed = false unless healthy lines << "CLI: `#{record.fetch('cli_version')}`. Dispatch: **#{record.fetch('state')}**." record.fetch("runs").each do |run| - name = [run.fetch("workflow"), run.dig("inputs", "case_id")].compact.join(" ") + name = workload_name(run) name = "[#{name}](https://github.com/#{repository}/actions/runs/#{run.fetch('id')})" if run["id"] lines << "- #{name}: **#{run.fetch('state')}**" end @@ -168,13 +177,18 @@ def write_summary(path, record) lines = ["## CLI canary benchmarks", "", "CLI: `#{record.fetch('cli_version')}`", "", "Dispatch: **#{record.fetch('state')}**. Benchmark outcomes are listed separately.", ""] record.fetch("runs").each do |run| - name = "#{run.fetch('repository')} / #{[run.fetch('workflow'), run.dig('inputs', 'case_id')].compact.join(' ')}" + name = "#{run.fetch('repository')} / #{workload_name(run)}" name = "[#{name}](#{run.fetch('url')})" if run["url"] lines << "- #{name}: **#{run.fetch('state')}**" end lines += ["", record["error"]] if record["error"] File.write(path, lines.join("\n") + "\n") end + + def workload_name(run) + inputs = run.fetch("inputs", {}) + [run.fetch("workflow"), *inputs.values_at("case_id", "variant", "cache_layer", "cache_profile")].compact.join(" ") + end end end diff --git a/suites/published.json b/suites/published.json index c9f3cbd49..ed7c9b625 100644 --- a/suites/published.json +++ b/suites/published.json @@ -44,9 +44,10 @@ "category": "docker", "step": "Docker build (server)", "workflow": "immich-immich-benchmark.yml", - "fresh_workflow": "immich-immich-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-immich", - "case_id": "immich" + "case_id": "immich", + "historical_fresh_workflow": "immich-fresh-benchmark.yml" }, { "benchmark": "mastodon-docker", @@ -64,9 +65,13 @@ "category": "docker", "step": "Docker build (Ruby+Node)", "workflow": "mastodon-mastodon-docker-benchmark.yml", - "fresh_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-mastodon", - "case_id": "mastodon" + "case_id": "mastodon", + "historical_fresh_workflow": "mastodon-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "server" + } }, { "benchmark": "mastodon-streaming", @@ -78,9 +83,13 @@ "category": "docker", "step": "Docker build (streaming service)", "workflow": "mastodon-mastodon-docker-benchmark.yml", - "fresh_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-mastodon", - "case_id": "mastodon" + "case_id": "mastodon", + "historical_fresh_workflow": "mastodon-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "streaming" + } }, { "benchmark": "discourse-image-factory-amd64", @@ -152,13 +161,17 @@ "category": "docker", "step": "Docker build (full stack)", "workflow": "posthog-posthog-benchmark.yml", - "fresh_workflow": "posthog-posthog-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "extra_providers": [ "boringcache-toolcache", "boringcache-mountcache" ], "historical_source_repo": "boringcache/benchmark-posthog", - "case_id": "posthog" + "case_id": "posthog", + "historical_fresh_workflow": "posthog-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "layers" + } }, { "benchmark": "storybook", @@ -254,7 +267,10 @@ "workflow": "zed-zed-cargo-rolling-chain.yml", "fresh_workflow": "zed-zed-cargo-product.yml", "historical_source_repo": "boringcache/benchmark-zed", - "case_id": "zed" + "case_id": "zed", + "fresh_inputs": { + "cache_layer": "combined" + } }, { "benchmark": "deno-cargo", @@ -271,7 +287,10 @@ "workflow": "deno-deno-cargo-rolling-chain.yml", "fresh_workflow": "deno-deno-cargo-product.yml", "historical_source_repo": "boringcache/benchmark-deno", - "case_id": "deno" + "case_id": "deno", + "fresh_inputs": { + "cache_profile": "cargo-product" + } }, { "benchmark": "duckgres", @@ -361,9 +380,13 @@ "category": "nodejs", "step": "Turbo build (pnpm monorepo)", "workflow": "n8n-n8n-benchmark.yml", - "fresh_workflow": "n8n-n8n-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-n8n", - "case_id": "n8n" + "case_id": "n8n", + "historical_fresh_workflow": "n8n-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "turbo" + } }, { "benchmark": "n8n-docker", @@ -375,9 +398,13 @@ "category": "docker", "step": "Docker build (n8n image)", "workflow": "n8n-n8n-docker-benchmark.yml", - "fresh_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-n8n", - "case_id": "n8n" + "case_id": "n8n", + "historical_fresh_workflow": "n8n-docker-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "docker" + } }, { "benchmark": "n8n-runners", @@ -389,9 +416,13 @@ "category": "docker", "step": "Docker build (task runners)", "workflow": "n8n-n8n-docker-benchmark.yml", - "fresh_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-n8n", - "case_id": "n8n" + "case_id": "n8n", + "historical_fresh_workflow": "n8n-docker-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "runners" + } }, { "benchmark": "n8n-runners-distroless", @@ -403,8 +434,12 @@ "category": "docker", "step": "Docker build (distroless task runners)", "workflow": "n8n-n8n-docker-benchmark.yml", - "fresh_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-n8n", - "case_id": "n8n" + "case_id": "n8n", + "historical_fresh_workflow": "n8n-docker-fresh-benchmark.yml", + "fresh_inputs": { + "variant": "distroless" + } } ] diff --git a/test/nightly_canaries_test.rb b/test/nightly_canaries_test.rb index db639b900..db0ed7291 100644 --- a/test/nightly_canaries_test.rb +++ b/test/nightly_canaries_test.rb @@ -83,6 +83,26 @@ def test_shared_workflow_dispatches_each_case_once_and_retains_its_selector assert posts.all? { |request| request[:body].dig("inputs", "cli_version") == VERSION } end + def test_shared_workflow_keeps_distinct_reviewed_variants_in_requests_and_receipts + first = {"source_repo" => "boringcache/benchmarks", "fresh_workflow" => "native-fresh-benchmark.yml", "case_id" => "n8n", "fresh_inputs" => {"variant" => "turbo"}} + second = first.merge("fresh_inputs" => {"variant" => "docker"}) + record = @runner.dispatch(repository: "boringcache/benchmarks", output: @output, summary: @summary, benchmarks: [first, first, second]) + posts = @runner.requests.select { |request| request[:body] } + assert_equal 2, posts.length + assert_equal %w[turbo docker], posts.map { |request| request[:body].dig("inputs", "variant") } + assert_equal %w[turbo docker], record["runs"].map { |run| run.dig("inputs", "variant") } + assert_includes File.read(@summary), "native-fresh-benchmark.yml n8n turbo" + assert_includes File.read(@summary), "native-fresh-benchmark.yml n8n docker" + end + + def test_invalid_workload_selectors_fail_before_any_network_or_build_request + item = {"source_repo" => "boringcache/benchmarks", "fresh_workflow" => "native-fresh-benchmark.yml", "case_id" => "n8n"} + [{"case_id" => "another-case"}, {"variant" => true}, {"cli_version" => VERSION}].each do |inputs| + assert_raises(NightlyCanaries::Error) { @runner.dispatch(repository: "boringcache/benchmarks", output: @output, benchmarks: [item.merge("fresh_inputs" => inputs)]) } + end + assert_empty @runner.requests + end + def test_incomplete_latest_canary_does_not_fall_back_to_an_older_one @runner.releases << @runner.releases.first.merge("tag_name" => "vcli-canary-abcdef012345", "published_at" => "2026-09-30T02:00:00Z", "assets" => []) assert_raises(NightlyCanaries::Error) { dispatch } diff --git a/test/publish_index_test.rb b/test/publish_index_test.rb index 51d8bb55a..05183bb46 100644 --- a/test/publish_index_test.rb +++ b/test/publish_index_test.rb @@ -33,6 +33,22 @@ def test_new_central_case_has_no_legacy_publication_feed assert_empty benchmark_workflow_runs(benchmark, "native-fresh-benchmark.yml", {}) end end + + def test_shared_canary_selectors_do_not_change_original_publication_workflows + expected = {"immich" => "immich-fresh-benchmark.yml", "mastodon-docker" => "mastodon-fresh-benchmark.yml", + "mastodon-streaming" => "mastodon-fresh-benchmark.yml", "posthog" => "posthog-fresh-benchmark.yml", + "n8n" => "n8n-fresh-benchmark.yml", "n8n-docker" => "n8n-docker-fresh-benchmark.yml", + "n8n-runners" => "n8n-docker-fresh-benchmark.yml", "n8n-runners-distroless" => "n8n-docker-fresh-benchmark.yml"} + expected.each do |id, workflow| + item = BENCHMARKS.find { |row| row.fetch("benchmark") == id } + calls = [] + stub(:latest_successful_runs, ->(repo:, workflow_name:) { calls << [repo, workflow_name]; [] }) do + benchmark_workflow_runs(item, item.fetch("fresh_workflow"), {}) + end + assert_equal [[item.fetch("historical_source_repo"), workflow]], calls + end + end + def test_migration_keeps_historical_execution_repository benchmark = {"source_repo" => "boringcache/benchmarks", "historical_source_repo" => "boringcache/benchmark-hugo", "case_id" => "hugo"} calls = [] From ebe9215a904081d279458722339ef85d2b742db6 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 20:33:31 +0100 Subject: [PATCH 09/17] Fix PostHog scope and output timing in shared comparisons --- .../actions/hugo-docker-benchmark/action.yml | 4 +- cases/hugo/recipe-review.md | 18 ++++++ .../posthog-docker-benchmark/action.yml | 38 +++-------- cases/posthog/recipe-review.md | 24 +++++++ .../image-output-qualification-02/series.json | 45 +++++++++++++ .../series.json | 46 +++++++++++++ .../image-output-qualification-02/series.json | 46 +++++++++++++ scripts/activate-docker-plan.rb | 5 ++ test/benchmark_case_scripts_test.rb | 16 +++++ test/native_case_test.rb | 64 ++++++++++++++++++- 10 files changed, 274 insertions(+), 32 deletions(-) create mode 100644 cases/hugo/recipe-review.md create mode 100644 cases/posthog/recipe-review.md create mode 100644 results/hugo/image-output-qualification-02/series.json create mode 100644 results/posthog/image-output-combined-qualification-01/series.json create mode 100644 results/posthog/image-output-qualification-02/series.json diff --git a/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml b/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml index 7bac6d2c1..03d1e0b97 100644 --- a/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml +++ b/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml @@ -117,8 +117,8 @@ runs: push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" load: "${{ inputs.load_image == 'true' }}" build-args: HUGO_BUILD_TAGS=extended,withdeploy - provenance: mode=max - sbom: true + provenance: "${{ inputs.load_image != 'true' && 'mode=max' || 'false' }}" + sbom: "${{ inputs.load_image != 'true' }}" cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - name: Publish with BoringCache diff --git a/cases/hugo/recipe-review.md b/cases/hugo/recipe-review.md new file mode 100644 index 000000000..9f0f2095f --- /dev/null +++ b/cases/hugo/recipe-review.md @@ -0,0 +1,18 @@ +# Hugo loaded-image comparison + +The [first image-output screening run](https://github.com/boringcache/benchmarks/actions/runs/37104595025) +failed in both providers with `docker exporter does not currently support exporting manifest lists`. +The recipe requested provenance and SBOM attestations while loading the image +into the runner's Docker image store. The failed request and its logs remain +evidence; it supplies no verified timing observations. + +The fresh loaded-image comparison disables provenance and SBOM attestations in +both providers. This is an explicit output projection from the upstream +publication recipe. The source, Dockerfile, build arguments, and image load are +unchanged. Publication paths retain their committed attestations. + +The plan activation regression checks the exact arguments for loaded and +unloaded output. The Actions recipe uses the same `load_image` condition. The +shared verifier inspects the selected local image after timing capture; this +establishes materialization, not runtime behavior. Corrected execution uses a +new predeclared series. diff --git a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml index 36db92e63..931361e86 100644 --- a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml +++ b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml @@ -132,7 +132,15 @@ runs: PLATFORM: "${{ inputs.platform }}" run: | set -euo pipefail - if [[ "$PHASE" == "warm" ]]; then + if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then + [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 + [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 + if [[ "$CACHE_LANE" == "rolling" ]]; then + cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" + else + cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" + fi + elif [[ "$PHASE" == "warm" ]]; then [[ -n "$PUBLISHED_SCOPE" ]] cache_scope="$PUBLISHED_SCOPE" else @@ -146,15 +154,6 @@ runs: cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" fi fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -255,25 +254,6 @@ runs: mode: docker trust-policy: restore fail-on-cache-miss: true - - name: Verify the completed image - if: inputs.load_image == 'true' - shell: bash - env: - IMAGE: "${{ inputs.strategy == 'boringcache' && 'posthog-benchmark:local' || - format('ghcr.io/{0}/posthog-benchmark:{1}', github.repository_owner, inputs.image_tag - || 'actions-cache') }}" - run: 'docker image inspect "$IMAGE" --format ''Image ID: {{.Id}}; size: {{.Size}} - bytes'' - - ' - - name: Verify the pushed image - if: inputs.push_image == 'true' - shell: bash - env: - IMAGE: ghcr.io/${{ github.repository_owner }}/posthog-benchmark:${{ inputs.image_tag - || (inputs.strategy == 'boringcache' && 'boringcache' || 'actions-cache') - }} - run: docker buildx imagetools inspect "$IMAGE" - name: Capture the build timing id: build_timing shell: bash diff --git a/cases/posthog/recipe-review.md b/cases/posthog/recipe-review.md new file mode 100644 index 000000000..b80fc5f39 --- /dev/null +++ b/cases/posthog/recipe-review.md @@ -0,0 +1,24 @@ +# PostHog shared execution review + +The native fresh comparison uses the pinned upstream Dockerfile for `layers` +and the declared tool-cache projection for `combined`. Both providers build, +export, and load the selected image. The shared output verifier inspects it after +the build timer stops. This check establishes image materialization, not runtime +application correctness. + +The [first image-output screening run](https://github.com/boringcache/benchmarks/actions/runs/37104600856) +completed both cold jobs, then failed both warm jobs because the legacy scope +assertion ran before the series scope was resolved. Its cold build timings also +include a superseded image inspection. Retain those original records as +diagnostic evidence; they do not satisfy the declared timing contract. + +For a declared series, resolve the existing deterministic scope before the +legacy warm branch. Cold and warm share case, series, sample, run, and attempt +identity. Legacy callers without a series still pass the cold scope explicitly, +and a warm call without that scope fails. Cache-miss enforcement is unchanged. + +The correction removes both old inspection steps. One shared verifier runs after +timing capture. Regression tests execute the action's scope shell with inputs +from the generated wrapper for both providers and variants, check the legacy +handoff, and require every Docker output inspection to follow timing capture. +Corrected execution requires a new predeclared series and a new harness ref. diff --git a/results/hugo/image-output-qualification-02/series.json b/results/hugo/image-output-qualification-02/series.json new file mode 100644 index 000000000..594017673 --- /dev/null +++ b/results/hugo/image-output-qualification-02/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "created_at": "2026-10-03T19:32:57Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "variant": null, + "workflow_inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485" +} diff --git a/results/posthog/image-output-combined-qualification-01/series.json b/results/posthog/image-output-combined-qualification-01/series.json new file mode 100644 index 000000000..f5c92c605 --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", + "created_at": "2026-10-03T19:32:57Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "variant": "combined", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad" +} diff --git a/results/posthog/image-output-qualification-02/series.json b/results/posthog/image-output-qualification-02/series.json new file mode 100644 index 000000000..763a6a4bf --- /dev/null +++ b/results/posthog/image-output-qualification-02/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-02", + "created_at": "2026-10-03T19:32:57Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc" +} diff --git a/scripts/activate-docker-plan.rb b/scripts/activate-docker-plan.rb index 805ab458a..a505269b4 100755 --- a/scripts/activate-docker-plan.rb +++ b/scripts/activate-docker-plan.rb @@ -34,6 +34,11 @@ case case_id when "hugo", "chroma", "linkerd2", "duckgres", "n8n" case case_id + when "hugo" + if load + replace.call('"--provenance", "mode=max"', '"--provenance", "false"') + replace.call('"--sbom", "true"', '"--sbom", "false"') + end when "linkerd2" replace.call('"LINKERD_VERSION=__SOURCE_TAG__"', JSON.generate("LINKERD_VERSION=#{args.fetch('source_tag')}")) when "duckgres" diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index e5d0b4e18..976b07803 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -100,6 +100,22 @@ def test_docker_output_check_uses_each_provider_image_and_rejects_failed_inspect end end + def test_hugo_loaded_image_disables_attestations_and_keeps_the_publication_recipe + %w[true false].each do |load| + with_case("hugo") do |directory| + _, errors, status = run_script(directory, "activate-docker-plan", "--load", load) + assert status.success?, errors + command = TomlRB.load_file(File.join(directory, ".boringcache.toml")).dig("adapters", "docker", "command") + assert_equal load == "true" ? "false" : "mode=max", command.fetch(command.index("--provenance") + 1) + assert_equal load == "true" ? "false" : "true", command.fetch(command.index("--sbom") + 1) + action = YAML.safe_load(File.read(File.join(directory, ".github/actions/hugo-docker-benchmark/action.yml"))) + build = action.dig("runs", "steps").find { |step| step["name"] == "Build with the GitHub Actions cache" }.fetch("with") + assert_equal "${{ inputs.load_image != 'true' && 'mode=max' || 'false' }}", build.fetch("provenance") + assert_equal "${{ inputs.load_image != 'true' }}", build.fetch("sbom") + end + end + end + def test_posthog_no_cache_option_preserves_its_declared_boolean_value with_case("posthog") do |directory| %w[false true].each do |value| diff --git a/test/native_case_test.rb b/test/native_case_test.rb index 0a5b8aad0..46d1ddbf5 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -157,7 +157,8 @@ def test_native_preparation_rejects_unverified_output_before_creating_a_checkout end def test_native_docker_variants_use_the_shared_loaded_image_check - {"mastodon" => %w[server server-sccache streaming], "posthog" => %w[layers combined], "immich" => [nil]}.each do |id, variants| + {"chroma" => [nil], "duckgres" => [nil], "hugo" => [nil], "linkerd2" => [nil], "n8n" => %w[docker runners distroless], + "mastodon" => %w[server server-sccache streaming], "posthog" => %w[layers combined], "immich" => [nil]}.each do |id, variants| variants.each do |variant| item = BenchmarkCases.load_case(id) assert_equal "native-fresh-benchmark.yml", BenchmarkCases.plan(item, variant: variant).fetch("workflow") @@ -166,11 +167,72 @@ def test_native_docker_variants_use_the_shared_loaded_image_check action = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, "cases", id, "payload", recipe.fetch("action"), "action.yml"))) steps = action.dig("runs", "steps") assert_operator steps.index { |step| step["id"] == "build_timing" }, :<, steps.index { |step| step["id"] == "output_verification" } + steps.each_with_index do |step, index| + next unless step["run"].to_s.match?(/\bdocker\s+(?:image|buildx\s+imagetools)\s+inspect\b|verify-docker-output\.rb/) + assert_operator steps.index { |entry| entry["id"] == "build_timing" }, :<, index, "#{id}: #{step['name']} must run after build timing" + end assert_operator steps.index { |step| step["id"] == "output_verification" }, :<, steps.index { |step| step["name"] == "Write the benchmark phase evidence" } end end end + def test_posthog_wrapper_resolves_the_same_series_scope_for_cold_and_warm + with_payload("posthog") do |item, directory| + %w[layers combined].each do |variant| + wrapper = NativeCase.write_action(item, directory: directory, lane: "fresh", variant: variant) + recipe = NativeCase.resolve(item.dig("execution", "native"), variant) + action = YAML.safe_load(File.read(File.join(directory, recipe.fetch("action"), "action.yml"))) + step = action.dig("runs", "steps").find { |entry| entry["id"] == "scope" } + defaults = action.fetch("inputs").transform_values { |input| input.fetch("default", "") } + %w[boringcache actions-cache].each do |provider| + scopes = %w[publish warm].map do |phase| + runtime = {"strategy" => provider, "phase" => phase, "benchmark_id" => recipe.fetch("benchmark_id")} + inputs = defaults.merge(wrapper.dig("runs", "steps", 0, "with").transform_values do |value| + value.gsub(/\$\{\{ inputs\.(\w+) \}\}/) { runtime.fetch(Regexp.last_match(1), "") } + end) + environment = step.fetch("env").transform_values do |value| + value.gsub(/\$\{\{ inputs\.(\w+) \}\}/) { inputs.fetch(Regexp.last_match(1)) } + end + assert_equal "", environment.fetch("PUBLISHED_SCOPE") + run_posthog_scope(item, directory, step, environment.merge("BENCHMARK_SERIES_ID" => "scope-screening-01", "BENCHMARK_SAMPLE" => "1")) + end + assert_equal scopes.first, scopes.last + assert_equal "#{recipe.fetch('benchmark_id')}-series-scope-screening-01-s1-r123-a2", scopes.first + end + end + end + end + + def test_posthog_legacy_warm_requires_and_keeps_the_explicit_published_scope + with_payload("posthog") do |item, directory| + action = YAML.safe_load(File.read(File.join(directory, item.dig("execution", "native", "action"), "action.yml"))) + step = action.dig("runs", "steps").find { |entry| entry["id"] == "scope" } + environment = {"BENCHMARK_ID" => "posthog", "CACHE_LANE" => "fresh", "PHASE" => "warm", "PLATFORM" => "linux/amd64", "PUBLISHED_SCOPE" => "seed-scope"} + assert_equal "seed-scope", run_posthog_scope(item, directory, step, environment) + assert_nil run_posthog_scope(item, directory, step, environment.merge("PUBLISHED_SCOPE" => ""), success: false) + end + end + + def run_posthog_scope(item, directory, step, environment, success: true) + FileUtils.cp(File.join(BenchmarkCases::ROOT, "scripts/scope-case-cache.rb"), File.join(directory, "scripts")) + File.write(File.join(directory, "benchmark-context.json"), JSON.generate(item)) + FileUtils.cp(File.join(BenchmarkCases::ROOT, "cases/posthog/payload/.boringcache.toml"), directory) + commands = File.join(directory, "commands") + FileUtils.mkdir_p(commands) + git = File.join(commands, "git") + File.write(git, "#!/bin/sh\ncase \"$*\" in\n 'config -f .gitmodules submodule.upstream.url') echo 'https://github.com/PostHog/posthog.git' ;;\n '-C upstream rev-parse HEAD') echo '#{item.dig('source', 'pins', 0, 'revision')}' ;;\n *) exit 1 ;;\nesac\n") + File.chmod(0o755, git) + output = File.join(directory, "scope-output") + File.write(output, "") + env = {"PATH" => "#{commands}:#{ENV.fetch('PATH')}", "GITHUB_OUTPUT" => output, + "GITHUB_RUN_ID" => "123", "GITHUB_RUN_ATTEMPT" => "2", "GITHUB_REF_NAME" => "main", + "BENCHMARK_SERIES_ID" => nil, "BENCHMARK_SAMPLE" => nil}.merge(environment) + _, errors, status = Open3.capture3(env, "bash", "-c", step.fetch("run"), chdir: directory) + assert_equal success, status.success?, errors + return unless status.success? + File.readlines(output).to_h { |line| line.strip.split("=", 2) }.fetch("cache_scope") + end + def test_unreviewed_recipe_input_and_unsafe_suffix_are_rejected with_payload("hugo-go") do |item, directory| assert_raises(NativeCase::Error) { NativeCase.write_action(item, directory: directory, lane: "fresh", suffix: "\nOTHER=value") } From a1742a327eaa0a0be2f5f911f217362224b57368 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 20:40:13 +0100 Subject: [PATCH 10/17] Preserve completed workload screens and corrected run requests --- data/latest/series.json | 1591 +++++++++++++---- docs/migration.md | 40 +- .../build-reuse-qualification-2026-10-03.json | 261 +++ .../docker-output-correction-2026-10-03.json | 137 ++ migration/evidence.json | 176 ++ ...image-output-qualification-2026-10-03.json | 85 + .../completions/37104573058.json | 13 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 847 +++++++++ .../image-output-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 166 ++ .../runs/1-actions-cache-warm.json | 166 ++ .../runs/1-boringcache-cold.json | 212 +++ .../runs/1-boringcache-warm.json | 212 +++ .../completions/37105483797.json | 13 + .../evidence.md | 16 + .../interpretation.md | 29 + .../report.json | 304 ++++ .../report.md | 22 + .../runs/1-boringcache-commit.json | 258 +++ .../completions/37105485279.json | 13 + .../evidence.md | 16 + .../interpretation.md | 28 + .../report.json | 259 +++ .../report.md | 22 + .../runs/1-boringcache-commit.json | 209 +++ .../completions/37104593175.json | 13 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 835 +++++++++ .../image-output-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 163 ++ .../runs/1-actions-cache-warm.json | 163 ++ .../runs/1-boringcache-cold.json | 209 +++ .../runs/1-boringcache-warm.json | 209 +++ .../completions/37104595025.json | 17 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 76 + .../image-output-qualification-01/report.md | 22 + .../1-native-fresh-benchmark.yml.json | 34 + .../completions/37104602471.json | 13 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 835 +++++++++ .../image-output-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 163 ++ .../runs/1-actions-cache-warm.json | 163 ++ .../runs/1-boringcache-cold.json | 209 +++ .../runs/1-boringcache-warm.json | 209 +++ .../completions/37104596700.json | 13 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 833 +++++++++ .../image-output-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 163 ++ .../runs/1-actions-cache-warm.json | 163 ++ .../runs/1-boringcache-cold.json | 208 +++ .../runs/1-boringcache-warm.json | 208 +++ .../completions/37104599574.json | 13 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 975 ++++++++++ .../image-output-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 198 ++ .../runs/1-actions-cache-warm.json | 198 ++ .../runs/1-boringcache-cold.json | 244 +++ .../runs/1-boringcache-warm.json | 244 +++ .../completions/37104598178.json | 13 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../image-output-qualification-01/report.json | 1079 +++++++++++ .../image-output-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 224 +++ .../runs/1-actions-cache-warm.json | 224 +++ .../runs/1-boringcache-cold.json | 270 +++ .../runs/1-boringcache-warm.json | 270 +++ .../1-native-fresh-benchmark.yml.json | 38 + .../completions/37104600856.json | 17 + .../image-output-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 527 ++++++ .../image-output-qualification-01/report.md | 29 + .../runs/1-actions-cache-cold.json | 191 ++ .../runs/1-boringcache-cold.json | 237 +++ .../1-native-fresh-benchmark.yml.json | 38 + .../completions/37105487087.json | 13 + .../build-reuse-qualification-01/evidence.md | 16 + .../interpretation.md | 11 + .../build-reuse-qualification-01/report.json | 846 +++++++++ .../build-reuse-qualification-01/report.md | 20 + .../runs/1-actions-cache-cold.json | 162 ++ .../runs/1-actions-cache-warm.json | 172 ++ .../runs/1-boringcache-cold.json | 207 +++ .../runs/1-boringcache-warm.json | 214 +++ 97 files changed, 16671 insertions(+), 382 deletions(-) create mode 100644 migration/build-reuse-qualification-2026-10-03.json create mode 100644 migration/docker-output-correction-2026-10-03.json create mode 100644 results/chroma/image-output-qualification-01/completions/37104573058.json create mode 100644 results/chroma/image-output-qualification-01/evidence.md create mode 100644 results/chroma/image-output-qualification-01/interpretation.md create mode 100644 results/chroma/image-output-qualification-01/report.json create mode 100644 results/chroma/image-output-qualification-01/report.md create mode 100644 results/chroma/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/chroma/image-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/chroma/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/chroma/image-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/completions/37105483797.json create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/evidence.md create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/interpretation.md create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/report.json create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/report.md create mode 100644 results/deno/rolling-cargo-product-seed-advance-01/runs/1-boringcache-commit.json create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/completions/37105485279.json create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/evidence.md create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/interpretation.md create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/report.json create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/report.md create mode 100644 results/deno/rolling-compiler-only-seed-advance-01/runs/1-boringcache-commit.json create mode 100644 results/duckgres/image-output-qualification-01/completions/37104593175.json create mode 100644 results/duckgres/image-output-qualification-01/evidence.md create mode 100644 results/duckgres/image-output-qualification-01/interpretation.md create mode 100644 results/duckgres/image-output-qualification-01/report.json create mode 100644 results/duckgres/image-output-qualification-01/report.md create mode 100644 results/duckgres/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/duckgres/image-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/duckgres/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/duckgres/image-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/hugo/image-output-qualification-01/completions/37104595025.json create mode 100644 results/hugo/image-output-qualification-01/evidence.md create mode 100644 results/hugo/image-output-qualification-01/interpretation.md create mode 100644 results/hugo/image-output-qualification-01/report.json create mode 100644 results/hugo/image-output-qualification-01/report.md create mode 100644 results/hugo/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/immich/image-output-qualification-01/completions/37104602471.json create mode 100644 results/immich/image-output-qualification-01/evidence.md create mode 100644 results/immich/image-output-qualification-01/interpretation.md create mode 100644 results/immich/image-output-qualification-01/report.json create mode 100644 results/immich/image-output-qualification-01/report.md create mode 100644 results/immich/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/immich/image-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/immich/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/immich/image-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/linkerd2/image-output-qualification-01/completions/37104596700.json create mode 100644 results/linkerd2/image-output-qualification-01/evidence.md create mode 100644 results/linkerd2/image-output-qualification-01/interpretation.md create mode 100644 results/linkerd2/image-output-qualification-01/report.json create mode 100644 results/linkerd2/image-output-qualification-01/report.md create mode 100644 results/linkerd2/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/linkerd2/image-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/linkerd2/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/linkerd2/image-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/mastodon/image-output-qualification-01/completions/37104599574.json create mode 100644 results/mastodon/image-output-qualification-01/evidence.md create mode 100644 results/mastodon/image-output-qualification-01/interpretation.md create mode 100644 results/mastodon/image-output-qualification-01/report.json create mode 100644 results/mastodon/image-output-qualification-01/report.md create mode 100644 results/mastodon/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/mastodon/image-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/mastodon/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/mastodon/image-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/n8n/image-output-qualification-01/completions/37104598178.json create mode 100644 results/n8n/image-output-qualification-01/evidence.md create mode 100644 results/n8n/image-output-qualification-01/interpretation.md create mode 100644 results/n8n/image-output-qualification-01/report.json create mode 100644 results/n8n/image-output-qualification-01/report.md create mode 100644 results/n8n/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/n8n/image-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/n8n/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/n8n/image-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/posthog/image-output-combined-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/posthog/image-output-qualification-01/completions/37104600856.json create mode 100644 results/posthog/image-output-qualification-01/evidence.md create mode 100644 results/posthog/image-output-qualification-01/interpretation.md create mode 100644 results/posthog/image-output-qualification-01/methodology-review.json create mode 100644 results/posthog/image-output-qualification-01/report.json create mode 100644 results/posthog/image-output-qualification-01/report.md create mode 100644 results/posthog/image-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/posthog/image-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/posthog/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/storybook/build-reuse-qualification-01/completions/37105487087.json create mode 100644 results/storybook/build-reuse-qualification-01/evidence.md create mode 100644 results/storybook/build-reuse-qualification-01/interpretation.md create mode 100644 results/storybook/build-reuse-qualification-01/report.json create mode 100644 results/storybook/build-reuse-qualification-01/report.md create mode 100644 results/storybook/build-reuse-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/storybook/build-reuse-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/storybook/build-reuse-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/storybook/build-reuse-qualification-01/runs/1-boringcache-warm.json diff --git a/data/latest/series.json b/data/latest/series.json index 59ebc69e1..cf34e5630 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -34,34 +34,70 @@ }, "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7", "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 4079, + "min": 4079, + "max": 4079 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 21, + "min": 21, + "max": 21 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 5097, + "min": 5097, + "max": 5097 + }, + "storage": { + "median": 2365446028, + "min": 2365446028, + "max": 2365446028 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 31, + "min": 31, + "max": 31 + }, + "storage": { + "median": 2365446028, + "min": 2365446028, + "max": 2365446028 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "chroma", @@ -81,14 +117,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37104573058" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104573058", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4ec2f8ff8a5307fe991773f741058c5ee838c844b9647d2807473a0eec5f67d9" + } + ], + "report_state": "current", + "report_path": "results/chroma/image-output-qualification-01/report.json", + "interpretation_path": "results/chroma/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104573058, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104573058.tar", + "bundle_bytes": 407040, + "sha256": "a2ebee35d448295a2e3b9a9b26eedd72efadd291a4845da08fb86fdafa2e95ba", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "chroma", @@ -700,22 +762,25 @@ }, "plan_sha256": "1fcb4e930c7efb0c363b76bb4a92c3ed26305ce8ddf087c465aa0481d009fbe9", "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, + "execution_verified": true, "methodology_issues": [ "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", "This series does not declare a comparison of provider performance." ], - "missing_observations": [ - [ - 1, - "commit", - "boringcache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } ], - "summaries": [], "dispatches": [ { "case_id": "deno", @@ -738,14 +803,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37105483797" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-cargo-product-seed-advance-01", + "sample": 1, + "run_id": "37105483797", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8cb31998c2a1e30dd44c1b3c2e154a355f5918e0afcc9c5d2c626755277c8d63" + } + ], + "report_state": "current", + "report_path": "results/deno/rolling-cargo-product-seed-advance-01/report.json", + "interpretation_path": "results/deno/rolling-cargo-product-seed-advance-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37105483797, + "inventory_state": "complete", + "verified_files": 9, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-build-reuse-2026-10-03/benchmarks-37105483797.tar", + "bundle_bytes": 319488, + "sha256": "8310584c8ddf50e5ae2905782dae9d74a7338b4c95cc592976c198eb6dad226d", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "deno", @@ -897,22 +988,29 @@ }, "plan_sha256": "184bb3883ff8ca58307b51e80cb069ecd6f0867f90dd2ead6492a78414585a26", "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, + "execution_verified": true, "methodology_issues": [ "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", "This series does not declare a comparison of provider performance." ], - "missing_observations": [ - [ - 1, - "commit", - "boringcache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": { + "median": 2085530204, + "min": 2085530204, + "max": 2085530204 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "deno", @@ -935,14 +1033,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37105485279" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-compiler-only-seed-advance-01", + "sample": 1, + "run_id": "37105485279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "89569d3d42cf58d029ba9647845e91ed77457624a109ab5d6bf6b70c342fa3d5" + } + ], + "report_state": "current", + "report_path": "results/deno/rolling-compiler-only-seed-advance-01/report.json", + "interpretation_path": "results/deno/rolling-compiler-only-seed-advance-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37105485279, + "inventory_state": "complete", + "verified_files": 9, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-build-reuse-2026-10-03/benchmarks-37105485279.tar", + "bundle_bytes": 327168, + "sha256": "b3c8161fad67811b144b654474c048277af884d088fe215d93788d95c0b61e06", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "deno", @@ -1694,61 +1818,123 @@ }, "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20", "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] - ], - "summaries": [], - "dispatches": [ + "missing_observations": [], + "summaries": [ { - "case_id": "duckgres", - "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", - "lane": "fresh", - "inputs": { - "case_id": "duckgres", - "cli_version": "v1.33.0", - "series_id": "image-output-qualification-01", - "sample": "1" + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 404, + "min": 404, + "max": 404 }, - "ref": "image-output-qualification-2026-10-03", - "state": "requested", - "requested_at": "2026-10-03T06:54:48Z", - "run_id": 37104593175, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175" - } - ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37104593175" - ] + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 18, + "min": 18, + "max": 18 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 193, + "min": 193, + "max": 193 + }, + "storage": { + "median": 1880170140, + "min": 1880170140, + "max": 1880170140 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 20, + "min": 20, + "max": 20 + }, + "storage": { + "median": 1880170140, + "min": 1880170140, + "max": 1880170140 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "duckgres", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:48Z", + "run_id": 37104593175, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104593175", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1e65a0a5ced80b50b20bf280565be3beae9686631ead32052c2963add521670" + } + ], + "report_state": "current", + "report_path": "results/duckgres/image-output-qualification-01/report.json", + "interpretation_path": "results/duckgres/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104593175, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104593175.tar", + "bundle_bytes": 355840, + "sha256": "22418e290a76b7cd5f32e6d07429b08d96119b7137aa9944a3f4bc3df53675cf", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "duckgres", @@ -2424,7 +2610,7 @@ }, "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700", "definition_sha256": "200bdce61b6cbba1361be02b435abbe9c4750e77dd72058281ba1c1e8a8882f4", - "state": "requested", + "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -2471,13 +2657,130 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025" } ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104595025", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "12aebc138373729492fa362c157080777851485d91d328d4d7a0209618719ce5" + } + ], + "report_state": "current", + "report_path": "results/hugo/image-output-qualification-01/report.json", + "interpretation_path": "results/hugo/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104595025, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104595025.tar", + "bundle_bytes": 235520, + "sha256": "af483c8478fcd8b726aff693fd2cea0281e91030e70a9af77218c6c3ad342f01", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "plan_path": "results/hugo/image-output-qualification-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:34:23Z", + "run_id": 37148378881, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], "runs_without_archives": [ - "37104595025" + "37148378881" ] }, { @@ -2626,34 +2929,70 @@ }, "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 371, + "min": 371, + "max": 371 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 57, + "min": 57, + "max": 57 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 207, + "min": 207, + "max": 207 + }, + "storage": { + "median": 1758920230, + "min": 1758920230, + "max": 1758920230 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 38, + "min": 38, + "max": 38 + }, + "storage": { + "median": 1758920230, + "min": 1758920230, + "max": 1758920230 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "immich", @@ -2673,14 +3012,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37104602471" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "immich", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104602471", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a232a9c86cf53c4945b672566ad89e76085267e61bdd5d7b0226606108e41521" + } + ], + "report_state": "current", + "report_path": "results/immich/image-output-qualification-01/report.json", + "interpretation_path": "results/immich/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104602471, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104602471.tar", + "bundle_bytes": 416256, + "sha256": "89dd58ddf791fbd4b2d66280cca759c2cff6f2f6bdbc4498ffeda57ed9185c87", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "linkerd2", @@ -2713,34 +3078,70 @@ }, "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 306, + "min": 306, + "max": 306 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 200, + "min": 200, + "max": 200 + }, + "storage": { + "median": 1230761384, + "min": 1230761384, + "max": 1230761384 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 12, + "min": 12, + "max": 12 + }, + "storage": { + "median": 1230761384, + "min": 1230761384, + "max": 1230761384 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "linkerd2", @@ -2760,14 +3161,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37104596700" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104596700", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3eba28538f46405b75d696f6c8071d56e26bf96e6b1be1f7fc5479893e282adf" + } + ], + "report_state": "current", + "report_path": "results/linkerd2/image-output-qualification-01/report.json", + "interpretation_path": "results/linkerd2/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104596700, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104596700.tar", + "bundle_bytes": 354816, + "sha256": "63efd14ac9e14ffd6b09b3ec57786f72aba962deda6f19fb773296b7428e1a60", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "linkerd2", @@ -2915,34 +3342,70 @@ }, "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 736, + "min": 736, + "max": 736 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 59, + "min": 59, + "max": 59 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 424, + "min": 424, + "max": 424 + }, + "storage": { + "median": 1130445165, + "min": 1130445165, + "max": 1130445165 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 38, + "min": 38, + "max": 38 + }, + "storage": { + "median": 1130445165, + "min": 1130445165, + "max": 1130445165 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "mastodon", @@ -2963,14 +3426,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37104599574" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104599574", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2e2e3c52be0fbdb8c2ea7924416d0bc1530b9ee9b391a13b8f3fc03b0ab61e52" + } + ], + "report_state": "current", + "report_path": "results/mastodon/image-output-qualification-01/report.json", + "interpretation_path": "results/mastodon/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104599574, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104599574.tar", + "bundle_bytes": 694784, + "sha256": "8b6a772ff39503d0e9aac23a96e13f627e9ffbba1f4e84ece45598857559ee60", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "n8n", @@ -3003,34 +3492,70 @@ }, "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 446, + "min": 446, + "max": 446 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 150, + "min": 150, + "max": 150 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 319, + "min": 319, + "max": 319 + }, + "storage": { + "median": 483433384, + "min": 483433384, + "max": 483433384 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 162, + "min": 162, + "max": 162 + }, + "storage": { + "median": 483433384, + "min": 483433384, + "max": 483433384 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "n8n", @@ -3051,14 +3576,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37104598178" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104598178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5f0af1311b0f43733e4562504aed6f42dde1420f1c800abe59315d5f97a55214" + } + ], + "report_state": "current", + "report_path": "results/n8n/image-output-qualification-01/report.json", + "interpretation_path": "results/n8n/image-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104598178, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104598178.tar", + "bundle_bytes": 1748480, + "sha256": "0bb9fc327d8fef9a8af1eb8bdcf8ec699508a46715bae610a28c12b0fc1dd49f", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "n8n", @@ -3773,14 +4324,255 @@ "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], - "runs_without_archives": [] + "runs_without_archives": [] + }, + { + "case_id": "opentelemetry-java", + "series_id": "migration-screening-01", + "plan_path": "results/opentelemetry-java/migration-screening-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "open-telemetry/opentelemetry-java", + "pins": [ + { + "path": "upstream", + "revision": "a674ff1acea16754a2ed413fb8966084220fbd8b", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "7d185a491dea81a5681ed1bf9f7b14e76c883193fee3d4441c63c37c533d1ecc", + "definition_sha256": "e1911882d495ee833991933d9b24cc819914f7b41072532fa8fe97e6d685c22a", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 966, + "min": 966, + "max": 966 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 171, + "min": 171, + "max": 171 + }, + "storage": { + "median": 50739407, + "min": 50739407, + "max": 50739407 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 975, + "min": 975, + "max": 975 + }, + "storage": { + "median": 51294699, + "min": 51294699, + "max": 51294699 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 150, + "min": 150, + "max": 150 + }, + "storage": { + "median": 51294699, + "min": 51294699, + "max": 51294699 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "opentelemetry-java", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opentelemetry-java", + "cli_version": "v1.33.0", + "series_id": "migration-screening-01", + "sample": "1" + }, + "ref": "active-case-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T01:14:41Z", + "run_id": 37085341225, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "opentelemetry-java", + "series_id": "migration-screening-01", + "sample": 1, + "run_id": "37085341225", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225", + "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e6bd8885e790a8be05bb2a0ed57a46aeb8ba5c8b40829917e5340a1e7d3a9f38" + } + ], + "report_state": "current", + "report_path": "results/opentelemetry-java/migration-screening-01/report.json", + "interpretation_path": "results/opentelemetry-java/migration-screening-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37085341225, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085341225.tar", + "bundle_bytes": 472576, + "sha256": "c5de94bda90ebb24e156895209fc648c5ad7d05cedbd7fb9cae1cd5ec67bff5c", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", + "plan_path": "results/posthog/image-output-combined-qualification-01/series.json", + "lane": "fresh", + "variant": "combined", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "image-output-combined-qualification-01", + "sample": "1" + }, + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:34:21Z", + "run_id": 37148377350, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + } + ], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [ + "37148377350" + ] }, { - "case_id": "opentelemetry-java", - "series_id": "migration-screening-01", - "plan_path": "results/opentelemetry-java/migration-screening-01/series.json", + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "plan_path": "results/posthog/image-output-qualification-01/series.json", "lane": "fresh", - "variant": null, + "variant": "layers", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -3788,141 +4580,128 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "open-telemetry/opentelemetry-java", + "repository": "PostHog/posthog", "pins": [ { "path": "upstream", - "revision": "a674ff1acea16754a2ed413fb8966084220fbd8b", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", "kind": "gitlink" } ] }, - "plan_sha256": "7d185a491dea81a5681ed1bf9f7b14e76c883193fee3d4441c63c37c533d1ecc", - "definition_sha256": "e1911882d495ee833991933d9b24cc819914f7b41072532fa8fe97e6d685c22a", - "state": "completed", + "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "state": "failed", "publication": "unreviewed", - "valid_for_comparison": true, - "execution_verified": true, - "methodology_issues": [], - "missing_observations": [], + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + ], + "missing_observations": [ + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], "summaries": [ { "phase": "cold", "provider": "actions-cache", "count": 1, "measurement": { - "median": 966, - "min": 966, - "max": 966 + "median": 1579, + "min": 1579, + "max": 1579 }, "storage": null, "storage_measured_count": 0 }, - { - "phase": "warm", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 171, - "min": 171, - "max": 171 - }, - "storage": { - "median": 50739407, - "min": 50739407, - "max": 50739407 - }, - "storage_measured_count": 1 - }, { "phase": "cold", "provider": "boringcache", "count": 1, "measurement": { - "median": 975, - "min": 975, - "max": 975 - }, - "storage": { - "median": 51294699, - "min": 51294699, - "max": 51294699 - }, - "storage_measured_count": 1 - }, - { - "phase": "warm", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 150, - "min": 150, - "max": 150 + "median": 813, + "min": 813, + "max": 813 }, "storage": { - "median": 51294699, - "min": 51294699, - "max": 51294699 + "median": 6237669195, + "min": 6237669195, + "max": 6237669195 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "opentelemetry-java", + "case_id": "posthog", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "opentelemetry-java", + "case_id": "posthog", "cli_version": "v1.33.0", - "series_id": "migration-screening-01", + "variant": "layers", + "series_id": "image-output-qualification-01", "sample": "1" }, - "ref": "active-case-qualification-2026-10-03", + "ref": "image-output-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-03T01:14:41Z", - "run_id": 37085341225, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225" + "requested_at": "2026-10-03T06:54:57Z", + "run_id": 37104600856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856" } ], "completions": [ { "schema_version": 1, - "case_id": "opentelemetry-java", - "series_id": "migration-screening-01", + "case_id": "posthog", + "series_id": "image-output-qualification-01", "sample": 1, - "run_id": "37085341225", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225", - "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "e6bd8885e790a8be05bb2a0ed57a46aeb8ba5c8b40829917e5340a1e7d3a9f38" + "run_id": "37104600856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog warm: failure", + "BoringCache posthog warm: failure" + ], + "evidence_manifest_sha256": "61aee72e30eb1e9f7e21dc0bf4c65e62b3d33de24057f5343593919036e6f320" } ], "report_state": "current", - "report_path": "results/opentelemetry-java/migration-screening-01/report.json", - "interpretation_path": "results/opentelemetry-java/migration-screening-01/interpretation.md", + "report_path": "results/posthog/image-output-qualification-01/report.json", + "interpretation_path": "results/posthog/image-output-qualification-01/interpretation.md", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37085341225, + "run_id": 37104600856, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 9, "gaps": [], "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085341225.tar", - "bundle_bytes": 472576, - "sha256": "c5de94bda90ebb24e156895209fc648c5ad7d05cedbd7fb9cae1cd5ec67bff5c", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104600856.tar", + "bundle_bytes": 392704, + "sha256": "6a38f37864533c9921020e3d994a21914adf337775f42b734e74fd894eebc7e2", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], @@ -3930,8 +4709,8 @@ }, { "case_id": "posthog", - "series_id": "image-output-qualification-01", - "plan_path": "results/posthog/image-output-qualification-01/series.json", + "series_id": "image-output-qualification-02", + "plan_path": "results/posthog/image-output-qualification-02/series.json", "lane": "fresh", "variant": "layers", "sample_count": 1, @@ -3957,8 +4736,8 @@ } ] }, - "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", - "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, @@ -3997,14 +4776,14 @@ "case_id": "posthog", "cli_version": "v1.33.0", "variant": "layers", - "series_id": "image-output-qualification-01", + "series_id": "image-output-qualification-02", "sample": "1" }, - "ref": "image-output-qualification-2026-10-03", + "ref": "docker-output-correction-2026-10-03", "state": "requested", - "requested_at": "2026-10-03T06:54:57Z", - "run_id": 37104600856, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + "requested_at": "2026-10-03T19:34:19Z", + "run_id": 37148375181, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181" } ], "completions": [], @@ -4013,7 +4792,7 @@ "interpretation_path": null, "evidence": [], "runs_without_archives": [ - "37104600856" + "37148375181" ] }, { @@ -4349,34 +5128,70 @@ }, "plan_sha256": "d463583c8fb964e5c3721430687e155f6579a1a423afa5508f758069e22942b5", "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 17, + "min": 17, + "max": 17 + }, + "storage": { + "median": 1226001, + "min": 1226001, + "max": 1226001 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 18, + "min": 18, + "max": 18 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 19, + "min": 19, + "max": 19 + }, + "storage": { + "median": 1212413, + "min": 1212413, + "max": 1212413 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "storybook", @@ -4396,14 +5211,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37105487087" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "storybook", + "series_id": "build-reuse-qualification-01", + "sample": 1, + "run_id": "37105487087", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "93bb5bb3faf15c087bb8fc3d967d8355c1246960603f6dc637057920c5b7c245" + } + ], + "report_state": "current", + "report_path": "results/storybook/build-reuse-qualification-01/report.json", + "interpretation_path": "results/storybook/build-reuse-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37105487087, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-build-reuse-2026-10-03/benchmarks-37105487087.tar", + "bundle_bytes": 509440, + "sha256": "512f07883a0dfbf61d823c68aca104b02abe640bc216f25f1b6d61805cf322bd", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "storybook", diff --git a/docs/migration.md b/docs/migration.md index 31d6d0163..9b9375770 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -45,16 +45,19 @@ as part of the reviewed caller cutover. PR validation runs the harness guards. The shared native fresh path also selects individual n8n Turbo/Docker, Mastodon server/streaming, and PostHog cache profiles from reviewed variant metadata; Immich's server uses the same path without a variant. Their imported matrices -remain registered diagnostics while output checks and live execution are -qualified. The new Docker paths do not establish a loaded or published image, -or qualify the original rolling publication behavior. Existing callers and -schedules still use the original repositories. +remain registered diagnostics. Loaded-image checks and live execution are +qualified per selected variant; they do not establish application runtime +behavior or qualify the original rolling publication behavior. Existing callers +and schedules still use the original repositories. Four initial screens (Chroma, Duckgres, Hugo Docker, and Linkerd2) were cancelled after review found missing image output checks. Their original plans, receipts, cancelled completions, and scoped evidence remain preserved. Their replacement fresh recipes load an image in both arms and use the shared output checker; -timing includes export and load. Those replacement definitions still require -live qualification. Original rolling publication projections remain diagnostic. +timing includes export and load. Chroma, Duckgres, and Linkerd2 completed their +replacement screens. Hugo's first replacement failed in both providers because +the Docker exporter rejected its attestation manifest list; a separate corrected +plan disables those attestations for loaded output in both arms. Original rolling +publication projections remain diagnostic. The same checker and load projection apply to the selected n8n Docker, Mastodon, PostHog, and Immich server recipes. Native preparation enforces the output requirement before creating a source checkout. No image load or application @@ -69,7 +72,30 @@ results retain their original timed scope and must not be mixed with this series Storybook's first fresh screen also timed dependency installation and sandbox creation in its setup metric. Its plan-bound methodology review prevents a comparative claim while retaining every original observation. The replacement -recipe stops the setup timer before those operations and needs a new series. +recipe stops the setup timer before those operations. Its new one-sample series +completed output and preserved completion checks, retaining the slower +BoringCache measurements. + +The [image-output archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) +preserves all eight frozen requests at `35e10981`, including the failed Hugo and +PostHog executions. Six selected paths completed cold/replay output and completion +checks: Chroma, Duckgres, Linkerd2, n8n Docker, Mastodon server, and Immich server. +The PostHog cold records also include inspection inside the timer, contrary to +their declared scope. A plan-bound methodology review prevents comparative use. +The [PostHog review](../cases/posthog/recipe-review.md) records the scope and timer +fixes; the [Hugo review](../cases/hugo/recipe-review.md) records its loaded-output +projection. New plans at signed `docker-output-correction-2026-10-03` request +PostHog layers, PostHog combined, and Hugo qualification without changing the +earlier declarations. + +The [build/reuse archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03) +preserves the corrected Storybook screen and both Deno seed-advance runs at +`b3ab6324`. Deno's publication-capable operations completed against the declared +parent cohorts, but remain diagnostic correctness investigations pending a generic +verified lineage contract. Native errors and unmeasured selected-tag storage +remain visible. Both archive releases were independently downloaded, checked +against archive digests, extracted, and verified against their scoped inventories. +One-sample screens do not approve publication, caller cutover, or deletion. GitHub's default CodeQL configuration now scans Ruby and Actions. Python was removed from its language list because this branch removes the maintained Python reporter; query suite, threat model, weekly schedule, and runner settings were diff --git a/migration/build-reuse-qualification-2026-10-03.json b/migration/build-reuse-qualification-2026-10-03.json new file mode 100644 index 000000000..38d2c22b3 --- /dev/null +++ b/migration/build-reuse-qualification-2026-10-03.json @@ -0,0 +1,261 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "publication": "unreviewed", + "scope": "Three predeclared one-sample requests. Backend telemetry does not replace original native counters, establish generic lineage, or qualify provider-performance claims.", + "runs": [ + { + "dispatch": { + "case_id": "deno", + "repository": "boringcache/benchmarks", + "workflow": "deno-deno-cargo-rolling-chain.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085348479-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "cargo-product", + "series_id": "rolling-cargo-product-seed-advance-01", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ], + "schema_version": 1, + "requested_at": "2026-10-03T07:10:39Z", + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "run_id": 37105483797, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797" + }, + "completion": { + "status": "success", + "github_conclusion": "success", + "errors": [] + }, + "observations": 1, + "valid_for_comparison": false, + "methodology_issues": [ + "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", + "This series does not declare a comparison of provider performance." + ] + }, + { + "dispatch": { + "case_id": "deno", + "repository": "boringcache/benchmarks", + "workflow": "deno-deno-cargo-rolling-chain.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "lane": "rolling", + "inputs": { + "cli_version": "v1.33.0", + "cache_scope": "r37085349929-a1", + "base_sha": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "head_sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "cache_profile": "compiler-only", + "series_id": "rolling-compiler-only-seed-advance-01", + "sample": "1" + }, + "phases": [ + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ], + "schema_version": 1, + "requested_at": "2026-10-03T07:10:41Z", + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "run_id": 37105485279, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279" + }, + "completion": { + "status": "success", + "github_conclusion": "success", + "errors": [] + }, + "observations": 1, + "valid_for_comparison": false, + "methodology_issues": [ + "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", + "This series does not declare a comparison of provider performance." + ] + }, + { + "dispatch": { + "case_id": "storybook", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "storybook", + "cli_version": "v1.33.0", + "series_id": "build-reuse-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T07:10:42Z", + "ref": "build-reuse-qualification-2026-10-03", + "state": "requested", + "run_id": 37105487087, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + }, + "completion": { + "status": "success", + "github_conclusion": "success", + "errors": [] + }, + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + } + ], + "backend_review": { + "schema_version": 1, + "workspace": "boringcache/benchmarks", + "scope": "Read-only backend telemetry joined to the original GitHub run and harness ref. Remote metrics do not replace native compiler counters or establish whole-run evidence completeness.", + "runs": [ + { + "run_id": 37105483797, + "generated_at": "2026-10-03T19:38:26Z", + "identity": { + "run_uid": "github-actions:boringcache/benchmarks:37105483797", + "run_label": "github-actions/37105483797#1", + "run_kind": "ci", + "provider": "github-actions", + "provider_run_uid": "37105483797", + "attempt": "1", + "repository": "boringcache/benchmarks", + "ref_type": "tag", + "ref_name": "build-reuse-qualification-2026-10-03", + "commit_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "summary_session_id": "proxy-summary-b9aff0d1-689a-4fe4-88c6-1a1e3d887f04", + "tools": [ + "archive", + "sccache" + ] + }, + "metrics": { + "session_count": 17, + "total_hits": 1837, + "total_misses": 59, + "total_errors": 0, + "total_bytes_read": 3855428334, + "total_bytes_written": 147697340, + "duration_seconds": 1922.6, + "read_label": "Remote read lookups", + "hit_label": "Remote hits", + "miss_label": "Missed lookups", + "hit_count_label": "remote hit", + "miss_count_label": "missed lookup" + }, + "cache_state": { + "state": "mostly_hot_with_misses", + "hit_rate": 96.9, + "hits": 1837, + "misses": 59, + "prior_cache_state": "usable_entry_existed" + }, + "service_side_issue": false, + "issue_candidates": [] + }, + { + "run_id": 37105485279, + "generated_at": "2026-10-03T19:38:27Z", + "identity": { + "run_uid": "github-actions:boringcache/benchmarks:37105485279", + "run_label": "github-actions/37105485279#1", + "run_kind": "ci", + "provider": "github-actions", + "provider_run_uid": "37105485279", + "attempt": "1", + "repository": "boringcache/benchmarks", + "ref_type": "tag", + "ref_name": "build-reuse-qualification-2026-10-03", + "commit_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "summary_session_id": "proxy-summary-a9442f57-2072-47e8-a0a6-ef21ac416c47", + "tools": [ + "sccache" + ] + }, + "metrics": { + "session_count": 9, + "total_hits": 3017, + "total_misses": 63, + "total_errors": 0, + "total_bytes_read": 1913624557, + "total_bytes_written": 88266847, + "duration_seconds": 1714.1, + "read_label": "Remote read lookups", + "hit_label": "Remote hits", + "miss_label": "Missed lookups", + "hit_count_label": "remote hit", + "miss_count_label": "missed lookup" + }, + "cache_state": { + "state": "mostly_hot_with_misses", + "hit_rate": 98, + "hits": 3017, + "misses": 63, + "prior_cache_state": "usable_entry_existed" + }, + "service_side_issue": false, + "issue_candidates": [] + } + ] + } +} diff --git a/migration/docker-output-correction-2026-10-03.json b/migration/docker-output-correction-2026-10-03.json new file mode 100644 index 000000000..3a8c31c48 --- /dev/null +++ b/migration/docker-output-correction-2026-10-03.json @@ -0,0 +1,137 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "publication": "unreviewed", + "scope": "Corrected PostHog and Hugo one-sample requests. This cache snapshot was collected after dispatch and is context, not per-phase occupancy or proof of absence of eviction. Original requests are unchanged.", + "cache_observations": { + "usage": { + "endpoint": "repos/boringcache/benchmarks/actions/cache/usage", + "collected_at": "2026-10-03T19:40:13Z", + "response": { + "full_name": "boringcache/benchmarks", + "active_caches_size_in_bytes": 10645321407, + "active_caches_count": 267 + } + }, + "storage-limit": { + "endpoint": "repos/boringcache/benchmarks/actions/cache/storage-limit", + "collected_at": "2026-10-03T19:40:13Z", + "response": { + "max_cache_size_gb": 10 + } + } + }, + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:34:19Z", + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "run_id": 37148375181, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + }, + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "image-output-combined-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:34:21Z", + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "run_id": 37148377350, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + }, + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T19:34:23Z", + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "run_id": 37148378881, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + } + ] +} diff --git a/migration/evidence.json b/migration/evidence.json index 050433016..69821c6c4 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -569,6 +569,182 @@ "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", "execution_qualification": "Passed scoped output and completion checks; one sample; publication and measurement interpretation remain unreviewed", "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104573058, + "verified_files": 13, + "gaps": [], + "sha256": "a2ebee35d448295a2e3b9a9b26eedd72efadd291a4845da08fb86fdafa2e95ba", + "bundle_bytes": 407040, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104573058.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104593175, + "verified_files": 13, + "gaps": [], + "sha256": "22418e290a76b7cd5f32e6d07429b08d96119b7137aa9944a3f4bc3df53675cf", + "bundle_bytes": 355840, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104593175.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104595025, + "verified_files": 7, + "gaps": [], + "sha256": "af483c8478fcd8b726aff693fd2cea0281e91030e70a9af77218c6c3ad342f01", + "bundle_bytes": 235520, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104595025.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Failed execution retained; this archive does not qualify the workload", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104596700, + "verified_files": 13, + "gaps": [], + "sha256": "63efd14ac9e14ffd6b09b3ec57786f72aba962deda6f19fb773296b7428e1a60", + "bundle_bytes": 354816, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104596700.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104598178, + "verified_files": 13, + "gaps": [], + "sha256": "0bb9fc327d8fef9a8af1eb8bdcf8ec699508a46715bae610a28c12b0fc1dd49f", + "bundle_bytes": 1748480, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104598178.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104599574, + "verified_files": 13, + "gaps": [], + "sha256": "8b6a772ff39503d0e9aac23a96e13f627e9ffbba1f4e84ece45598857559ee60", + "bundle_bytes": 694784, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104599574.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104600856, + "verified_files": 9, + "gaps": [], + "sha256": "6a38f37864533c9921020e3d994a21914adf337775f42b734e74fd894eebc7e2", + "bundle_bytes": 392704, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104600856.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Failed execution retained; this archive does not qualify the workload", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37104602471, + "verified_files": 13, + "gaps": [], + "sha256": "89dd58ddf791fbd4b2d66280cca759c2cff6f2f6bdbc4498ffeda57ed9185c87", + "bundle_bytes": 416256, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104602471.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37105483797, + "verified_files": 9, + "gaps": [], + "sha256": "8310584c8ddf50e5ae2905782dae9d74a7338b4c95cc592976c198eb6dad226d", + "bundle_bytes": 319488, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-build-reuse-2026-10-03/benchmarks-37105483797.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; diagnostic methodology limits remain; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37105485279, + "verified_files": 9, + "gaps": [], + "sha256": "b3c8161fad67811b144b654474c048277af884d088fe215d93788d95c0b61e06", + "bundle_bytes": 327168, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-build-reuse-2026-10-03/benchmarks-37105485279.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; diagnostic methodology limits remain; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37105487087, + "verified_files": 15, + "gaps": [], + "sha256": "512f07883a0dfbf61d823c68aca104b02abe640bc216f25f1b6d61805cf322bd", + "bundle_bytes": 509440, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-build-reuse-2026-10-03/benchmarks-37105487087.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-03", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" } ] } diff --git a/migration/image-output-qualification-2026-10-03.json b/migration/image-output-qualification-2026-10-03.json index 659f89768..f4d930648 100644 --- a/migration/image-output-qualification-2026-10-03.json +++ b/migration/image-output-qualification-2026-10-03.json @@ -138,5 +138,90 @@ "sample": "1" } } + ], + "outcomes": [ + { + "run_id": 37104573058, + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "variant": null, + "completion": "success", + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + }, + { + "run_id": 37104593175, + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "variant": null, + "completion": "success", + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + }, + { + "run_id": 37104595025, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "variant": null, + "completion": "failed", + "observations": 0, + "valid_for_comparison": false, + "methodology_issues": [] + }, + { + "run_id": 37104602471, + "case_id": "immich", + "series_id": "image-output-qualification-01", + "variant": null, + "completion": "success", + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + }, + { + "run_id": 37104596700, + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "variant": null, + "completion": "success", + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + }, + { + "run_id": 37104599574, + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "variant": "server", + "completion": "success", + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + }, + { + "run_id": 37104598178, + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "variant": "docker", + "completion": "success", + "observations": 4, + "valid_for_comparison": true, + "methodology_issues": [] + }, + { + "run_id": 37104600856, + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "variant": "layers", + "completion": "failed", + "observations": 2, + "valid_for_comparison": false, + "methodology_issues": [ + "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + ] + } ] } diff --git a/results/chroma/image-output-qualification-01/completions/37104573058.json b/results/chroma/image-output-qualification-01/completions/37104573058.json new file mode 100644 index 000000000..ac9cdcf48 --- /dev/null +++ b/results/chroma/image-output-qualification-01/completions/37104573058.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104573058", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4ec2f8ff8a5307fe991773f741058c5ee838c844b9647d2807473a0eec5f67d9" +} diff --git a/results/chroma/image-output-qualification-01/evidence.md b/results/chroma/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..8e3e5c0e9 --- /dev/null +++ b/results/chroma/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104573058 | 407040 | `a2ebee35d448295a2e3b9a9b26eedd72efadd291a4845da08fb86fdafa2e95ba` | 13 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/chroma/image-output-qualification-01/interpretation.md b/results/chroma/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..3e2bf8806 --- /dev/null +++ b/results/chroma/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# chroma: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104573058) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both providers completed the declared cold and identical-source replay, including export and load. Logs and original phase records establish that the shared verifier inspected the selected local image after timing capture. This establishes image materialization, not runtime application correctness. Slower observations remain in the report. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/chroma/image-output-qualification-01/report.json b/results/chroma/image-output-qualification-01/report.json new file mode 100644 index 000000000..99a4f1d19 --- /dev/null +++ b/results/chroma/image-output-qualification-01/report.json @@ -0,0 +1,847 @@ +{ + "schema_version": 1, + "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104573058", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104573058", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4ec2f8ff8a5307fe991773f741058c5ee838c844b9647d2807473a0eec5f67d9" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 4079, + "min": 4079, + "max": 4079 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 21, + "min": 21, + "max": 21 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 5097, + "min": 5097, + "max": 5097 + }, + "storage": { + "median": 2365446028, + "min": 2365446028, + "max": 2365446028 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 31, + "min": 31, + "max": 31 + }, + "storage": { + "median": 2365446028, + "min": 2365446028, + "max": 2365446028 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "chroma", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 4072, + "workflow_seconds": 4082, + "total_seconds": 4079, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 4079 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "chroma-series-image-output-qualification-01-s1-r37104573058-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170360", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] + }, + { + "schema_version": 1, + "benchmark": "chroma", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 13, + "workflow_seconds": 24, + "total_seconds": 21, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 21 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "chroma-series-image-output-qualification-01-s1-r37104573058-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170505", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] + }, + { + "schema_version": 1, + "benchmark": "chroma", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 5097, + "workflow_seconds": 5102, + "total_seconds": 5097, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 5097 + }, + "cache": { + "hit": null, + "sccache_proof": true, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + }, + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 2365446028, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 2365446028, + "unmeasured_tags": [], + "total_bytes": 2365446028, + "observations": [ + { + "requested_tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e2f910d8-c942-48d2-bed3-d62d6930583b", + "manifest_root_digest": "sha256:933a7e55badbd0063d05542d4c457aaed5fb9985f57a93efd4bbcb51f854a850", + "kv_total_size": 0, + "compressed_size": 2365446028, + "size": 2365446028 + } + ] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170359", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] + }, + { + "schema_version": 1, + "benchmark": "chroma", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 31, + "workflow_seconds": 35, + "total_seconds": 31, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 31 + }, + "cache": { + "hit": null, + "sccache_proof": true, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 2365446028, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 2365446028, + "unmeasured_tags": [], + "total_bytes": 2365446028, + "observations": [ + { + "requested_tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e2f910d8-c942-48d2-bed3-d62d6930583b", + "manifest_root_digest": "sha256:933a7e55badbd0063d05542d4c457aaed5fb9985f57a93efd4bbcb51f854a850", + "kv_total_size": 0, + "compressed_size": 2365446028, + "size": 2365446028 + } + ] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170504", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] + } + ], + "exclusions": [] +} diff --git a/results/chroma/image-output-qualification-01/report.md b/results/chroma/image-output-qualification-01/report.md new file mode 100644 index 000000000..d9ab6a439 --- /dev/null +++ b/results/chroma/image-output-qualification-01/report.md @@ -0,0 +1,20 @@ +# chroma: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 4079 | 4079–4079 | unmeasured | 0 | +| warm | actions-cache | 1 | 21 | 21–21 | unmeasured | 0 | +| cold | boringcache | 1 | 5097 | 5097–5097 | 2365446028 | 1 | +| warm | boringcache | 1 | 31 | 31–31 | 2365446028 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/chroma/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/chroma/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..ae866bc14 --- /dev/null +++ b/results/chroma/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,166 @@ +{ + "schema_version": 1, + "benchmark": "chroma", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 4072, + "workflow_seconds": 4082, + "total_seconds": 4079, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 4079 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "chroma-series-image-output-qualification-01-s1-r37104573058-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170360", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] +} diff --git a/results/chroma/image-output-qualification-01/runs/1-actions-cache-warm.json b/results/chroma/image-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..bbfa8781e --- /dev/null +++ b/results/chroma/image-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,166 @@ +{ + "schema_version": 1, + "benchmark": "chroma", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 13, + "workflow_seconds": 24, + "total_seconds": 21, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 21 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "chroma-series-image-output-qualification-01-s1-r37104573058-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170505", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] +} diff --git a/results/chroma/image-output-qualification-01/runs/1-boringcache-cold.json b/results/chroma/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..4a378fca6 --- /dev/null +++ b/results/chroma/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,212 @@ +{ + "schema_version": 1, + "benchmark": "chroma", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 5097, + "workflow_seconds": 5102, + "total_seconds": 5097, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 5097 + }, + "cache": { + "hit": null, + "sccache_proof": true, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + }, + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 2365446028, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 2365446028, + "unmeasured_tags": [], + "total_bytes": 2365446028, + "observations": [ + { + "requested_tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e2f910d8-c942-48d2-bed3-d62d6930583b", + "manifest_root_digest": "sha256:933a7e55badbd0063d05542d4c457aaed5fb9985f57a93efd4bbcb51f854a850", + "kv_total_size": 0, + "compressed_size": 2365446028, + "size": 2365446028 + } + ] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170359", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] +} diff --git a/results/chroma/image-output-qualification-01/runs/1-boringcache-warm.json b/results/chroma/image-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..e62516a42 --- /dev/null +++ b/results/chroma/image-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,212 @@ +{ + "schema_version": 1, + "benchmark": "chroma", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "chroma", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", + "source": { + "repository": "chroma-core/chroma", + "pins": [ + { + "path": "upstream", + "revision": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-chroma", + "branch": "main", + "revision": "edf3931449fd113176a09059c5d8c0f5df9ec03f" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "chroma" + }, + "historical_workflow": "chroma-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 1, + "native": { + "action": ".github/actions/chroma-docker-benchmark", + "benchmark_id": "chroma", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "chroma-docker-local": "{scope}-docker", + "chroma-sccache-local": "{scope}-sccache" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 31, + "workflow_seconds": 35, + "total_seconds": 31, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 31 + }, + "cache": { + "hit": null, + "sccache_proof": true, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 2365446028, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 2365446028, + "unmeasured_tags": [], + "total_bytes": 2365446028, + "observations": [ + { + "requested_tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "tag": "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e2f910d8-c942-48d2-bed3-d62d6930583b", + "manifest_root_digest": "sha256:933a7e55badbd0063d05542d4c457aaed5fb9985f57a93efd4bbcb51f854a850", + "kv_total_size": 0, + "compressed_size": 2365446028, + "size": 2365446028 + } + ] + } + }, + "source": { + "repository": "chroma-core/chroma", + "sha": "848cf58dd6dd952f8d0ff672af7b30cbf4d67f81" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "chroma-series-image-output-qualification-01-s1-r37104573058-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104573058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170504", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104573058-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Fresh cold and warm builds use the unchanged pinned source; diagnostic source-layer invalidation is excluded", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] +} diff --git a/results/deno/rolling-cargo-product-seed-advance-01/completions/37105483797.json b/results/deno/rolling-cargo-product-seed-advance-01/completions/37105483797.json new file mode 100644 index 000000000..db4e8d63c --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/completions/37105483797.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-cargo-product-seed-advance-01", + "sample": 1, + "run_id": "37105483797", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8cb31998c2a1e30dd44c1b3c2e154a355f5918e0afcc9c5d2c626755277c8d63" +} diff --git a/results/deno/rolling-cargo-product-seed-advance-01/evidence.md b/results/deno/rolling-cargo-product-seed-advance-01/evidence.md new file mode 100644 index 000000000..4a5711dbe --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37105483797 | 319488 | `8310584c8ddf50e5ae2905782dae9d74a7338b4c95cc592976c198eb6dad226d` | 9 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Output and completion checks passed; diagnostic methodology limits remain; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/deno/rolling-cargo-product-seed-advance-01/interpretation.md b/results/deno/rolling-cargo-product-seed-advance-01/interpretation.md new file mode 100644 index 000000000..772c1e906 --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/interpretation.md @@ -0,0 +1,29 @@ +# deno: rolling-cargo-product-seed-advance-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37105483797) executed its frozen request at `b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e`. + +This adjacent-source publication-capable run retained the declared parent cold cohort. Output and preserved completion checks passed. It remains a single-provider correctness investigation; a generic verified seed/source lineage contract is still absent. Native product error counters and exact publication evidence require the separate review below. Do not present this as provider-performance evidence. + +The [generated report](report.md) retains 1 original phase records. Completion is `success`; comparative use is `false`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The original product evidence reports publication trust with `write_allowed: true`, +a completed Cargo command, and compiler reuse. Native tool counters are: + +| Build | Hits | Misses | Cache errors | Write errors | Read errors | Timeouts | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| deno / denort / test_server | 1334 | 52 | 5 | 0 | 0 | 0 | +| denort_desktop | 500 | 3 | 0 | 0 | 0 | 0 | + +These nonzero native cache errors remain part of the result. Successful binaries +and backend reuse do not establish error-free native operation. The earlier +restore-only consumer's write errors remain in its separate original records; +the publication-capable run does not replace or correct them. + +The selected-tag storage probe measured a subtotal of 4,653,493,676 bytes. +The Git database tag was unmeasured, so total storage remains null. Preserved +logs show three archive entries saved after each Cargo operation. This supports +publication to the retained cohort; it does not qualify all future source changes. diff --git a/results/deno/rolling-cargo-product-seed-advance-01/report.json b/results/deno/rolling-cargo-product-seed-advance-01/report.json new file mode 100644 index 000000000..674cd2379 --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/report.json @@ -0,0 +1,304 @@ +{ + "schema_version": 1, + "plan_sha256": "1fcb4e930c7efb0c363b76bb4a92c3ed26305ce8ddf087c465aa0481d009fbe9", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-cargo-product-seed-advance-01", + "sample": 1, + "run_id": "37105483797", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105483797", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "8cb31998c2a1e30dd44c1b3c2e154a355f5918e0afcc9c5d2c626755277c8d63" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "deno-cargo", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "cargo-product", + "mode": "cargo", + "adapter": "sccache", + "case": { + "schema_version": 1, + "case_id": "deno", + "workspace": "boringcache/benchmarks", + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-deno", + "branch": "main", + "revision": "a9dc6263ba3e32b3c0d2e8666ad9e06cb37d4055" + }, + "verification": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ], + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/deno-deno-cargo-product.yml", + "variant_input": "cache_profile", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ] + }, + { + "path": ".github/workflows/deno-deno-cargo-rolling-chain.yml", + "variant_input": "cache_profile", + "lane": "rolling", + "inputs": {}, + "variants": [ + "cargo-product", + "compiler-only" + ] + } + ], + "sync": "adjacent-pair", + "sync_minute": 2, + "source_prefix": "DENO" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 1924, + "workflow_seconds": 1979, + "total_seconds": 1924, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "build_and_reuse_seconds": 1924 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "deno-cargo-target-r37085348479-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64" + ], + "complete": false, + "measured_bytes": 4653493676, + "unmeasured_tags": [ + "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "be27c238-34e4-40f3-b61e-d327233950c3", + "manifest_root_digest": "sha256:75f59ceb17c1d9a2af0647213f9845b864daa4bb0745ba014669f312129f2ec8", + "kv_total_size": 0, + "compressed_size": 147646264, + "size": 147646264 + }, + { + "requested_tag": "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b5f51953-1156-46e1-913c-e19941d72353", + "manifest_root_digest": "sha256:8c67396313fcf02704726ff625adbf920528a01ad8d8ea0fa2ff2d221f6c2230", + "kv_total_size": 0, + "compressed_size": 3598961, + "size": 3598961 + }, + { + "requested_tag": "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "status": "miss", + "kv_total_size": 0 + }, + { + "requested_tag": "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "01f70013-9330-4077-8633-d28810fbfbb6", + "manifest_root_digest": "sha256:cba7c2d6ecb99f147346196ed3d57e6132abece4e8b83481b553b8b9ff128b72", + "kv_total_size": 0, + "compressed_size": 2416718989, + "size": 2416718989 + }, + { + "requested_tag": "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 2085529462, + "compressed_size": 2085529462, + "size": 2085529462 + } + ] + } + }, + "source": { + "repository": "denoland/deno", + "sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "cargo", + "resolved_tags": [ + "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105483797", + "run_attempt": "1", + "job": "advance", + "workflow": "deno: Deno Cargo rolling run", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170391", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105483797-1", + "series": { + "id": "rolling-cargo-product-seed-advance-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105483797" + ] + } + ], + "exclusions": [] +} diff --git a/results/deno/rolling-cargo-product-seed-advance-01/report.md b/results/deno/rolling-cargo-product-seed-advance-01/report.md new file mode 100644 index 000000000..88db8b71e --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/report.md @@ -0,0 +1,22 @@ +# deno: rolling-cargo-product-seed-advance-01 + +Question: Does the pinned workload produce verified output and reuse its declared cache? + +Measured scope: The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison. + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| commit | boringcache | 1 | unmeasured | 0 | + +Methodology prevents a comparative claim: + +- Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic. +- This series does not declare a comparison of provider performance. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/deno/rolling-cargo-product-seed-advance-01/runs/1-boringcache-commit.json b/results/deno/rolling-cargo-product-seed-advance-01/runs/1-boringcache-commit.json new file mode 100644 index 000000000..396c8a9cb --- /dev/null +++ b/results/deno/rolling-cargo-product-seed-advance-01/runs/1-boringcache-commit.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "benchmark": "deno-cargo", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "cargo-product", + "mode": "cargo", + "adapter": "sccache", + "case": { + "schema_version": 1, + "case_id": "deno", + "workspace": "boringcache/benchmarks", + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-deno", + "branch": "main", + "revision": "a9dc6263ba3e32b3c0d2e8666ad9e06cb37d4055" + }, + "verification": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ], + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/deno-deno-cargo-product.yml", + "variant_input": "cache_profile", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ] + }, + { + "path": ".github/workflows/deno-deno-cargo-rolling-chain.yml", + "variant_input": "cache_profile", + "lane": "rolling", + "inputs": {}, + "variants": [ + "cargo-product", + "compiler-only" + ] + } + ], + "sync": "adjacent-pair", + "sync_minute": 2, + "source_prefix": "DENO" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 1924, + "workflow_seconds": 1979, + "total_seconds": 1924, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "build_and_reuse_seconds": 1924 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "deno-cargo-target-r37085348479-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64" + ], + "complete": false, + "measured_bytes": 4653493676, + "unmeasured_tags": [ + "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "be27c238-34e4-40f3-b61e-d327233950c3", + "manifest_root_digest": "sha256:75f59ceb17c1d9a2af0647213f9845b864daa4bb0745ba014669f312129f2ec8", + "kv_total_size": 0, + "compressed_size": 147646264, + "size": 147646264 + }, + { + "requested_tag": "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b5f51953-1156-46e1-913c-e19941d72353", + "manifest_root_digest": "sha256:8c67396313fcf02704726ff625adbf920528a01ad8d8ea0fa2ff2d221f6c2230", + "kv_total_size": 0, + "compressed_size": 3598961, + "size": 3598961 + }, + { + "requested_tag": "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "status": "miss", + "kv_total_size": 0 + }, + { + "requested_tag": "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "01f70013-9330-4077-8633-d28810fbfbb6", + "manifest_root_digest": "sha256:cba7c2d6ecb99f147346196ed3d57e6132abece4e8b83481b553b8b9ff128b72", + "kv_total_size": 0, + "compressed_size": 2416718989, + "size": 2416718989 + }, + { + "requested_tag": "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64", + "tag": "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 2085529462, + "compressed_size": 2085529462, + "size": 2085529462 + } + ] + } + }, + "source": { + "repository": "denoland/deno", + "sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "cargo", + "resolved_tags": [ + "deno-cargo-registry-cache-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-registry-index-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-git-db-r37085348479-a1-ubuntu-24-x86_64", + "deno-cargo-target-r37085348479-a1-ubuntu-24-x86_64", + "deno-rust-cache-r37085348479-a1-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105483797", + "run_attempt": "1", + "job": "advance", + "workflow": "deno: Deno Cargo rolling run", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170391", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105483797-1", + "series": { + "id": "rolling-cargo-product-seed-advance-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105483797" + ] +} diff --git a/results/deno/rolling-compiler-only-seed-advance-01/completions/37105485279.json b/results/deno/rolling-compiler-only-seed-advance-01/completions/37105485279.json new file mode 100644 index 000000000..f6f5c84f6 --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/completions/37105485279.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-compiler-only-seed-advance-01", + "sample": 1, + "run_id": "37105485279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "89569d3d42cf58d029ba9647845e91ed77457624a109ab5d6bf6b70c342fa3d5" +} diff --git a/results/deno/rolling-compiler-only-seed-advance-01/evidence.md b/results/deno/rolling-compiler-only-seed-advance-01/evidence.md new file mode 100644 index 000000000..b9f1148c0 --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37105485279 | 327168 | `b3c8161fad67811b144b654474c048277af884d088fe215d93788d95c0b61e06` | 9 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Output and completion checks passed; diagnostic methodology limits remain; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/deno/rolling-compiler-only-seed-advance-01/interpretation.md b/results/deno/rolling-compiler-only-seed-advance-01/interpretation.md new file mode 100644 index 000000000..b3df230fd --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/interpretation.md @@ -0,0 +1,28 @@ +# deno: rolling-compiler-only-seed-advance-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37105485279) executed its frozen request at `b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e`. + +This adjacent-source publication-capable run retained the declared parent cold cohort. Output and preserved completion checks passed. It remains a single-provider correctness investigation; a generic verified seed/source lineage contract is still absent. Native product error counters and exact publication evidence require the separate review below. Do not present this as provider-performance evidence. + +The [generated report](report.md) retains 1 original phase records. Completion is `success`; comparative use is `false`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The original product evidence reports publication trust with `write_allowed: true`, +a completed Cargo command, and compiler reuse. Native tool counters are: + +| Build | Hits | Misses | Cache errors | Write errors | Read errors | Timeouts | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| deno / denort / test_server | 2484 | 52 | 10 | 0 | 0 | 0 | +| denort_desktop | 533 | 3 | 0 | 0 | 0 | 0 | + +These nonzero native cache errors remain part of the result. Successful binaries +and backend reuse do not establish error-free native operation. The earlier +restore-only consumer's write errors remain in its separate original records; +the publication-capable run does not replace or correct them. + +The selected compiler-cache tag measured 2,085,530,204 bytes. No target or +dependency archive was selected for this profile. This is a provider-reported +compiler-cache measurement, not a total comparison against another provider. diff --git a/results/deno/rolling-compiler-only-seed-advance-01/report.json b/results/deno/rolling-compiler-only-seed-advance-01/report.json new file mode 100644 index 000000000..5f19f0471 --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/report.json @@ -0,0 +1,259 @@ +{ + "schema_version": 1, + "plan_sha256": "184bb3883ff8ca58307b51e80cb069ecd6f0867f90dd2ead6492a78414585a26", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic.", + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "deno", + "series_id": "rolling-compiler-only-seed-advance-01", + "sample": 1, + "run_id": "37105485279", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105485279", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "89569d3d42cf58d029ba9647845e91ed77457624a109ab5d6bf6b70c342fa3d5" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "commit", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": { + "median": 2085530204, + "min": 2085530204, + "max": 2085530204 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "deno-cargo", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "compiler-only", + "mode": "cargo", + "adapter": "sccache", + "case": { + "schema_version": 1, + "case_id": "deno", + "workspace": "boringcache/benchmarks", + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-deno", + "branch": "main", + "revision": "a9dc6263ba3e32b3c0d2e8666ad9e06cb37d4055" + }, + "verification": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ], + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/deno-deno-cargo-product.yml", + "variant_input": "cache_profile", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ] + }, + { + "path": ".github/workflows/deno-deno-cargo-rolling-chain.yml", + "variant_input": "cache_profile", + "lane": "rolling", + "inputs": {}, + "variants": [ + "cargo-product", + "compiler-only" + ] + } + ], + "sync": "adjacent-pair", + "sync_minute": 2, + "source_prefix": "DENO" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 1715, + "workflow_seconds": 1766, + "total_seconds": 1715, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "build_and_reuse_seconds": 1715 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "deno-rust-cache-r37085349929-a1-compiler-only", + "workspace": "boringcache/benchmarks", + "storage_bytes": 2085530204, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 2085530204, + "unmeasured_tags": [], + "total_bytes": 2085530204, + "observations": [ + { + "requested_tag": "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64", + "tag": "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 2085530204, + "compressed_size": 2085530204, + "size": 2085530204 + } + ] + } + }, + "source": { + "repository": "denoland/deno", + "sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "cargo", + "resolved_tags": [ + "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105485279", + "run_attempt": "1", + "job": "advance", + "workflow": "deno: Deno Cargo rolling run", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170459", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105485279-1", + "series": { + "id": "rolling-compiler-only-seed-advance-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105485279" + ] + } + ], + "exclusions": [] +} diff --git a/results/deno/rolling-compiler-only-seed-advance-01/report.md b/results/deno/rolling-compiler-only-seed-advance-01/report.md new file mode 100644 index 000000000..05d3ccc2b --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/report.md @@ -0,0 +1,22 @@ +# deno: rolling-compiler-only-seed-advance-01 + +Question: Does the pinned workload produce verified output and reuse its declared cache? + +Measured scope: The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison. + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| commit | boringcache | 1 | 2085530204 | 1 | + +Methodology prevents a comparative claim: + +- Rolling observations have no verified seed and changed-source sequence; retain them as diagnostic. +- This series does not declare a comparison of provider performance. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/deno/rolling-compiler-only-seed-advance-01/runs/1-boringcache-commit.json b/results/deno/rolling-compiler-only-seed-advance-01/runs/1-boringcache-commit.json new file mode 100644 index 000000000..8ef8f4544 --- /dev/null +++ b/results/deno/rolling-compiler-only-seed-advance-01/runs/1-boringcache-commit.json @@ -0,0 +1,209 @@ +{ + "schema_version": 1, + "benchmark": "deno-cargo", + "strategy": "boringcache", + "lane": "rolling", + "phase": "commit", + "variant": "compiler-only", + "mode": "cargo", + "adapter": "sccache", + "case": { + "schema_version": 1, + "case_id": "deno", + "workspace": "boringcache/benchmarks", + "definition_sha256": "fe3d0639e3684a12c63b1599ce267083d1ece052b5d76f13eb72cf628c448506", + "source": { + "repository": "denoland/deno", + "pins": [ + { + "path": "payload/benchmark-source.env", + "revision": "b8681dfa6442a0dfa5c7ddc43b279c504079cf2a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "b4f08f127652d8442b4d3dbabc277aca3840bc1d", + "kind": "env" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-deno", + "branch": "main", + "revision": "a9dc6263ba3e32b3c0d2e8666ad9e06cb37d4055" + }, + "verification": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ], + "comparison": { + "question": "Does the pinned workload produce verified output and reuse its declared cache?", + "providers": [ + "boringcache" + ], + "timed_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "primary_metric": "correctness", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/deno-deno-cargo-product.yml", + "variant_input": "cache_profile", + "lane": "fresh", + "inputs": {}, + "phases": [ + "cold", + "commit" + ], + "variants": [ + "cargo-product", + "compiler-only" + ] + }, + { + "path": ".github/workflows/deno-deno-cargo-rolling-chain.yml", + "variant_input": "cache_profile", + "lane": "rolling", + "inputs": {}, + "variants": [ + "cargo-product", + "compiler-only" + ] + } + ], + "sync": "adjacent-pair", + "sync_minute": 2, + "source_prefix": "DENO" + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 1715, + "workflow_seconds": 1766, + "total_seconds": 1715, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "The two BoringCache Cargo release operations, including restore, build, and any permitted publication. Fresh changed-source consumers use restore-only trust; rolling jobs permit publication to the declared cohort. Output verification and unrelated setup are excluded. This is a correctness proof, not a provider timing comparison.", + "build_and_reuse_seconds": 1715 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "deno-rust-cache-r37085349929-a1-compiler-only", + "workspace": "boringcache/benchmarks", + "storage_bytes": 2085530204, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 2085530204, + "unmeasured_tags": [], + "total_bytes": 2085530204, + "observations": [ + { + "requested_tag": "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64", + "tag": "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 2085530204, + "compressed_size": 2085530204, + "size": 2085530204 + } + ] + } + }, + "source": { + "repository": "denoland/deno", + "sha": "b4f08f127652d8442b4d3dbabc277aca3840bc1d" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "cargo", + "resolved_tags": [ + "deno-rust-cache-r37085349929-a1-compiler-only-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105485279", + "run_attempt": "1", + "job": "advance", + "workflow": "deno: Deno Cargo rolling run", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170459", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105485279-1", + "series": { + "id": "rolling-compiler-only-seed-advance-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify Deno Linux release recipe", + "Require a clean product starting point", + "Verify the Deno base build", + "Require an empty destination", + "Verify the Deno head build", + "Require one-commit source cadence", + "Verify the Deno build" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105485279" + ] +} diff --git a/results/duckgres/image-output-qualification-01/completions/37104593175.json b/results/duckgres/image-output-qualification-01/completions/37104593175.json new file mode 100644 index 000000000..43612f22f --- /dev/null +++ b/results/duckgres/image-output-qualification-01/completions/37104593175.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104593175", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1e65a0a5ced80b50b20bf280565be3beae9686631ead32052c2963add521670" +} diff --git a/results/duckgres/image-output-qualification-01/evidence.md b/results/duckgres/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..af755cd27 --- /dev/null +++ b/results/duckgres/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104593175 | 355840 | `22418e290a76b7cd5f32e6d07429b08d96119b7137aa9944a3f4bc3df53675cf` | 13 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/duckgres/image-output-qualification-01/interpretation.md b/results/duckgres/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..1aed6fd8f --- /dev/null +++ b/results/duckgres/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# duckgres: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104593175) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both providers completed the declared cold and identical-source replay, including export and load. Logs and original phase records establish that the shared verifier inspected the selected local image after timing capture. This establishes image materialization, not runtime application correctness. Slower observations remain in the report. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/duckgres/image-output-qualification-01/report.json b/results/duckgres/image-output-qualification-01/report.json new file mode 100644 index 000000000..0f2f1e3e1 --- /dev/null +++ b/results/duckgres/image-output-qualification-01/report.json @@ -0,0 +1,835 @@ +{ + "schema_version": 1, + "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104593175", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104593175", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c1e65a0a5ced80b50b20bf280565be3beae9686631ead32052c2963add521670" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 404, + "min": 404, + "max": 404 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 18, + "min": 18, + "max": 18 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 193, + "min": 193, + "max": 193 + }, + "storage": { + "median": 1880170140, + "min": 1880170140, + "max": 1880170140 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 20, + "min": 20, + "max": 20 + }, + "storage": { + "median": 1880170140, + "min": 1880170140, + "max": 1880170140 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 398, + "workflow_seconds": 408, + "total_seconds": 404, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 404 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170507", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] + }, + { + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 13, + "workflow_seconds": 21, + "total_seconds": 18, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 18 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170515", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] + }, + { + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 193, + "workflow_seconds": 196, + "total_seconds": 193, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 193 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + }, + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1880170140, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1880170140, + "unmeasured_tags": [], + "total_bytes": 1880170140, + "observations": [ + { + "requested_tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "797f8085-b829-4bcf-b793-376fdd74eab6", + "manifest_root_digest": "sha256:7dcb4920e87f9e7142c4cf8e307bfec9f6122010ff30edb17546938fc2f40e4a", + "kv_total_size": 0, + "compressed_size": 1880170140, + "size": 1880170140 + } + ] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170508", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] + }, + { + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 20, + "workflow_seconds": 24, + "total_seconds": 20, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 20 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1880170140, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1880170140, + "unmeasured_tags": [], + "total_bytes": 1880170140, + "observations": [ + { + "requested_tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "797f8085-b829-4bcf-b793-376fdd74eab6", + "manifest_root_digest": "sha256:7dcb4920e87f9e7142c4cf8e307bfec9f6122010ff30edb17546938fc2f40e4a", + "kv_total_size": 0, + "compressed_size": 1880170140, + "size": 1880170140 + } + ] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170514", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] + } + ], + "exclusions": [] +} diff --git a/results/duckgres/image-output-qualification-01/report.md b/results/duckgres/image-output-qualification-01/report.md new file mode 100644 index 000000000..4f246054d --- /dev/null +++ b/results/duckgres/image-output-qualification-01/report.md @@ -0,0 +1,20 @@ +# duckgres: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 404 | 404–404 | unmeasured | 0 | +| warm | actions-cache | 1 | 18 | 18–18 | unmeasured | 0 | +| cold | boringcache | 1 | 193 | 193–193 | 1880170140 | 1 | +| warm | boringcache | 1 | 20 | 20–20 | 1880170140 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/duckgres/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/duckgres/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..b4b2d1879 --- /dev/null +++ b/results/duckgres/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 398, + "workflow_seconds": 408, + "total_seconds": 404, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 404 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170507", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] +} diff --git a/results/duckgres/image-output-qualification-01/runs/1-actions-cache-warm.json b/results/duckgres/image-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..a6fdd8f3b --- /dev/null +++ b/results/duckgres/image-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 13, + "workflow_seconds": 21, + "total_seconds": 18, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 18 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170515", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] +} diff --git a/results/duckgres/image-output-qualification-01/runs/1-boringcache-cold.json b/results/duckgres/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..0d9a50bc8 --- /dev/null +++ b/results/duckgres/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,209 @@ +{ + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 193, + "workflow_seconds": 196, + "total_seconds": 193, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 193 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + }, + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1880170140, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1880170140, + "unmeasured_tags": [], + "total_bytes": 1880170140, + "observations": [ + { + "requested_tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "797f8085-b829-4bcf-b793-376fdd74eab6", + "manifest_root_digest": "sha256:7dcb4920e87f9e7142c4cf8e307bfec9f6122010ff30edb17546938fc2f40e4a", + "kv_total_size": 0, + "compressed_size": 1880170140, + "size": 1880170140 + } + ] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170508", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] +} diff --git a/results/duckgres/image-output-qualification-01/runs/1-boringcache-warm.json b/results/duckgres/image-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..30b79a7ae --- /dev/null +++ b/results/duckgres/image-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,209 @@ +{ + "schema_version": 1, + "benchmark": "duckgres", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "duckgres", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", + "source": { + "repository": "PostHog/duckgres", + "pins": [ + { + "path": "upstream", + "revision": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-duckgres", + "branch": "main", + "revision": "43060b6b39cff5aecd2138b6cf99571f6876d98b" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "duckgres" + }, + "historical_workflow": "duckgres-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 5, + "native": { + "action": ".github/actions/duckgres-docker-benchmark", + "benchmark_id": "duckgres", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "duckgres-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 20, + "workflow_seconds": 24, + "total_seconds": 20, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 20 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1880170140, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1880170140, + "unmeasured_tags": [], + "total_bytes": 1880170140, + "observations": [ + { + "requested_tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "tag": "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "797f8085-b829-4bcf-b793-376fdd74eab6", + "manifest_root_digest": "sha256:7dcb4920e87f9e7142c4cf8e307bfec9f6122010ff30edb17546938fc2f40e4a", + "kv_total_size": 0, + "compressed_size": 1880170140, + "size": 1880170140 + } + ] + } + }, + "source": { + "repository": "PostHog/duckgres", + "sha": "12e26cf3a900823cf415d826ab2a9b8f0d57b2a3" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "duckgres-series-image-output-qualification-01-s1-r37104593175-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104593175", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170514", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104593175-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] +} diff --git a/results/hugo/image-output-qualification-01/completions/37104595025.json b/results/hugo/image-output-qualification-01/completions/37104595025.json new file mode 100644 index 000000000..fbfc30dd9 --- /dev/null +++ b/results/hugo/image-output-qualification-01/completions/37104595025.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104595025", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "12aebc138373729492fa362c157080777851485d91d328d4d7a0209618719ce5" +} diff --git a/results/hugo/image-output-qualification-01/evidence.md b/results/hugo/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..aae9e0e60 --- /dev/null +++ b/results/hugo/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104595025 | 235520 | `af483c8478fcd8b726aff693fd2cea0281e91030e70a9af77218c6c3ad342f01` | 7 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Failed execution retained; this archive does not qualify the workload. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/hugo/image-output-qualification-01/interpretation.md b/results/hugo/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..b56e70cd3 --- /dev/null +++ b/results/hugo/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# hugo: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104595025) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both cold builds failed because the Docker exporter cannot export the requested attestation manifest list. No original verified phase JSON exists, so no measurements were reconstructed. Corrected loaded-image execution disables provenance and SBOM attestations in both providers and uses a new plan. + +The [generated report](report.md) retains 0 original phase records. Completion is `failed`; comparative use is `false`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/hugo/image-output-qualification-01/report.json b/results/hugo/image-output-qualification-01/report.json new file mode 100644 index 000000000..9a470a707 --- /dev/null +++ b/results/hugo/image-output-qualification-01/report.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104595025", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "12aebc138373729492fa362c157080777851485d91d328d4d7a0209618719ce5" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104595025", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "12aebc138373729492fa362c157080777851485d91d328d4d7a0209618719ce5" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/hugo/image-output-qualification-01/report.md b/results/hugo/image-output-qualification-01/report.md new file mode 100644 index 000000000..b3146b654 --- /dev/null +++ b/results/hugo/image-output-qualification-01/report.md @@ -0,0 +1,22 @@ +# hugo: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: 4 declared observations missing; 0 failed. Publication requires review. + +Execution: unqualified; missing or failed job completion checks. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | + +Run 37104595025 failed completion checks: + +- Workflow concluded failure +- BoringCache hugo cold: failure +- GitHub Actions hugo cold: failure + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/hugo/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json b/results/hugo/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..c97f0f443 --- /dev/null +++ b/results/hugo/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-03T19:34:23Z", + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "run_id": 37148378881, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881" +} diff --git a/results/immich/image-output-qualification-01/completions/37104602471.json b/results/immich/image-output-qualification-01/completions/37104602471.json new file mode 100644 index 000000000..ba586b5d7 --- /dev/null +++ b/results/immich/image-output-qualification-01/completions/37104602471.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "immich", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104602471", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a232a9c86cf53c4945b672566ad89e76085267e61bdd5d7b0226606108e41521" +} diff --git a/results/immich/image-output-qualification-01/evidence.md b/results/immich/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..89d10a071 --- /dev/null +++ b/results/immich/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104602471 | 416256 | `89dd58ddf791fbd4b2d66280cca759c2cff6f2f6bdbc4498ffeda57ed9185c87` | 13 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/immich/image-output-qualification-01/interpretation.md b/results/immich/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..040477161 --- /dev/null +++ b/results/immich/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# immich: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104602471) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both providers completed the declared cold and identical-source replay, including export and load. Logs and original phase records establish that the shared verifier inspected the selected local image after timing capture. This establishes image materialization, not runtime application correctness. Slower observations remain in the report. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/immich/image-output-qualification-01/report.json b/results/immich/image-output-qualification-01/report.json new file mode 100644 index 000000000..f545bd75a --- /dev/null +++ b/results/immich/image-output-qualification-01/report.json @@ -0,0 +1,835 @@ +{ + "schema_version": 1, + "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "immich", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104602471", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a232a9c86cf53c4945b672566ad89e76085267e61bdd5d7b0226606108e41521" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 371, + "min": 371, + "max": 371 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 57, + "min": 57, + "max": 57 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 207, + "min": 207, + "max": 207 + }, + "storage": { + "median": 1758920230, + "min": 1758920230, + "max": 1758920230 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 38, + "min": 38, + "max": 38 + }, + "storage": { + "median": 1758920230, + "min": 1758920230, + "max": 1758920230 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 363, + "workflow_seconds": 378, + "total_seconds": 371, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 371 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170611", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 50, + "workflow_seconds": 62, + "total_seconds": 57, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 57 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170620", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 207, + "workflow_seconds": 213, + "total_seconds": 207, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 207 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + }, + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1758920230, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1758920230, + "unmeasured_tags": [], + "total_bytes": 1758920230, + "observations": [ + { + "requested_tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "a76c6543-c983-4ce3-92bb-706077ad7f54", + "manifest_root_digest": "sha256:e31c9fd2e3cd65ec01f0d76023be83ba3ce9dd74e397edba07489a17496ce3c9", + "kv_total_size": 0, + "compressed_size": 1758920230, + "size": 1758920230 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170612", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] + }, + { + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 38, + "workflow_seconds": 44, + "total_seconds": 38, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 38 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1758920230, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1758920230, + "unmeasured_tags": [], + "total_bytes": 1758920230, + "observations": [ + { + "requested_tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "a76c6543-c983-4ce3-92bb-706077ad7f54", + "manifest_root_digest": "sha256:e31c9fd2e3cd65ec01f0d76023be83ba3ce9dd74e397edba07489a17496ce3c9", + "kv_total_size": 0, + "compressed_size": 1758920230, + "size": 1758920230 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170621", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] + } + ], + "exclusions": [] +} diff --git a/results/immich/image-output-qualification-01/report.md b/results/immich/image-output-qualification-01/report.md new file mode 100644 index 000000000..a806b9c7b --- /dev/null +++ b/results/immich/image-output-qualification-01/report.md @@ -0,0 +1,20 @@ +# immich: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 371 | 371–371 | unmeasured | 0 | +| warm | actions-cache | 1 | 57 | 57–57 | unmeasured | 0 | +| cold | boringcache | 1 | 207 | 207–207 | 1758920230 | 1 | +| warm | boringcache | 1 | 38 | 38–38 | 1758920230 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/immich/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/immich/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..3d5afff2a --- /dev/null +++ b/results/immich/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 363, + "workflow_seconds": 378, + "total_seconds": 371, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 371 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170611", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] +} diff --git a/results/immich/image-output-qualification-01/runs/1-actions-cache-warm.json b/results/immich/image-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..e43feb41a --- /dev/null +++ b/results/immich/image-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 50, + "workflow_seconds": 62, + "total_seconds": 57, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 57 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170620", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] +} diff --git a/results/immich/image-output-qualification-01/runs/1-boringcache-cold.json b/results/immich/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..4a18c2dec --- /dev/null +++ b/results/immich/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,209 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 207, + "workflow_seconds": 213, + "total_seconds": 207, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 207 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + }, + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1758920230, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1758920230, + "unmeasured_tags": [], + "total_bytes": 1758920230, + "observations": [ + { + "requested_tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "a76c6543-c983-4ce3-92bb-706077ad7f54", + "manifest_root_digest": "sha256:e31c9fd2e3cd65ec01f0d76023be83ba3ce9dd74e397edba07489a17496ce3c9", + "kv_total_size": 0, + "compressed_size": 1758920230, + "size": 1758920230 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170612", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] +} diff --git a/results/immich/image-output-qualification-01/runs/1-boringcache-warm.json b/results/immich/image-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..06c4f46b9 --- /dev/null +++ b/results/immich/image-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,209 @@ +{ + "schema_version": 1, + "benchmark": "immich-server", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "immich", + "workspace": "boringcache/benchmarks", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "source": { + "repository": "immich-app/immich", + "pins": [ + { + "path": "upstream", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-immich", + "branch": "main", + "revision": "ec958a4d8bc0c427319922c00ed973a201235025" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "immich" + }, + "historical_workflow": "immich-immich-fresh-benchmark.yml" + }, + { + "path": ".github/workflows/immich-immich-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/immich-immich-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 12, + "cache_tags": { + "immich-docker-local": "{scope}-docker", + "immich-ccache-local": "{scope}-ccache" + }, + "native": { + "action": ".github/actions/immich-docker-benchmark", + "benchmark_id": "immich-server", + "fresh_inputs": { + "load_image": "true" + }, + "rolling_inputs": {} + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 38, + "workflow_seconds": 44, + "total_seconds": 38, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 38 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1758920230, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1758920230, + "unmeasured_tags": [], + "total_bytes": 1758920230, + "observations": [ + { + "requested_tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "tag": "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "a76c6543-c983-4ce3-92bb-706077ad7f54", + "manifest_root_digest": "sha256:e31c9fd2e3cd65ec01f0d76023be83ba3ce9dd74e397edba07489a17496ce3c9", + "kv_total_size": 0, + "compressed_size": 1758920230, + "size": 1758920230 + } + ] + } + }, + "source": { + "repository": "immich-app/immich", + "sha": "007c18aec6cc9552fb7d8b7c791c41d7424876b0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "immich-server-series-image-output-qualification-01-s1-r37104602471-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104602471", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170621", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104602471-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] +} diff --git a/results/linkerd2/image-output-qualification-01/completions/37104596700.json b/results/linkerd2/image-output-qualification-01/completions/37104596700.json new file mode 100644 index 000000000..a10393ca5 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/completions/37104596700.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104596700", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3eba28538f46405b75d696f6c8071d56e26bf96e6b1be1f7fc5479893e282adf" +} diff --git a/results/linkerd2/image-output-qualification-01/evidence.md b/results/linkerd2/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..529d8b1a9 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104596700 | 354816 | `63efd14ac9e14ffd6b09b3ec57786f72aba962deda6f19fb773296b7428e1a60` | 13 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/linkerd2/image-output-qualification-01/interpretation.md b/results/linkerd2/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..afc528124 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# linkerd2: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104596700) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both providers completed the declared cold and identical-source replay, including export and load. Logs and original phase records establish that the shared verifier inspected the selected local image after timing capture. This establishes image materialization, not runtime application correctness. Slower observations remain in the report. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/linkerd2/image-output-qualification-01/report.json b/results/linkerd2/image-output-qualification-01/report.json new file mode 100644 index 000000000..4ed7af69c --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/report.json @@ -0,0 +1,833 @@ +{ + "schema_version": 1, + "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104596700", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3eba28538f46405b75d696f6c8071d56e26bf96e6b1be1f7fc5479893e282adf" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 306, + "min": 306, + "max": 306 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 200, + "min": 200, + "max": 200 + }, + "storage": { + "median": 1230761384, + "min": 1230761384, + "max": 1230761384 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 12, + "min": 12, + "max": 12 + }, + "storage": { + "median": 1230761384, + "min": 1230761384, + "max": 1230761384 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 298, + "workflow_seconds": 308, + "total_seconds": 306, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 306 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170523", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] + }, + { + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 5, + "workflow_seconds": 17, + "total_seconds": 15, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 15 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170535", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] + }, + { + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 200, + "workflow_seconds": 202, + "total_seconds": 200, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 200 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + }, + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1230761384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 1230761384, + "unmeasured_tags": [], + "total_bytes": 1230761384, + "observations": [ + { + "requested_tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b542bd96-10d5-4849-96bc-e547137715ba", + "manifest_root_digest": "sha256:edb5fa1af4afd692a6cd8e570386819e7a508ccfa7b3d081483fd8440a631072", + "kv_total_size": 0, + "compressed_size": 1230761384, + "size": 1230761384 + } + ] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170524", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] + }, + { + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 12, + "workflow_seconds": 15, + "total_seconds": 12, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 12 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": null + }, + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1230761384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 1230761384, + "unmeasured_tags": [], + "total_bytes": 1230761384, + "observations": [ + { + "requested_tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b542bd96-10d5-4849-96bc-e547137715ba", + "manifest_root_digest": "sha256:edb5fa1af4afd692a6cd8e570386819e7a508ccfa7b3d081483fd8440a631072", + "kv_total_size": 0, + "compressed_size": 1230761384, + "size": 1230761384 + } + ] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170536", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] + } + ], + "exclusions": [] +} diff --git a/results/linkerd2/image-output-qualification-01/report.md b/results/linkerd2/image-output-qualification-01/report.md new file mode 100644 index 000000000..1f70b17dd --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/report.md @@ -0,0 +1,20 @@ +# linkerd2: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 306 | 306–306 | unmeasured | 0 | +| warm | actions-cache | 1 | 15 | 15–15 | unmeasured | 0 | +| cold | boringcache | 1 | 200 | 200–200 | 1230761384 | 1 | +| warm | boringcache | 1 | 12 | 12–12 | 1230761384 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/linkerd2/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/linkerd2/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..3caf43f67 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 298, + "workflow_seconds": 308, + "total_seconds": 306, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 306 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170523", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] +} diff --git a/results/linkerd2/image-output-qualification-01/runs/1-actions-cache-warm.json b/results/linkerd2/image-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..dfe15e183 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 5, + "workflow_seconds": 17, + "total_seconds": 15, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 15 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170535", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] +} diff --git a/results/linkerd2/image-output-qualification-01/runs/1-boringcache-cold.json b/results/linkerd2/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..17ab8f6a6 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,208 @@ +{ + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 200, + "workflow_seconds": 202, + "total_seconds": 200, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 200 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + }, + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1230761384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 1230761384, + "unmeasured_tags": [], + "total_bytes": 1230761384, + "observations": [ + { + "requested_tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b542bd96-10d5-4849-96bc-e547137715ba", + "manifest_root_digest": "sha256:edb5fa1af4afd692a6cd8e570386819e7a508ccfa7b3d081483fd8440a631072", + "kv_total_size": 0, + "compressed_size": 1230761384, + "size": 1230761384 + } + ] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170524", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] +} diff --git a/results/linkerd2/image-output-qualification-01/runs/1-boringcache-warm.json b/results/linkerd2/image-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..94b9d272e --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,208 @@ +{ + "schema_version": 1, + "benchmark": "linkerd2-v2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "linkerd2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "source": { + "repository": "linkerd/linkerd2", + "pins": [ + { + "path": "upstream", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-linkerd2", + "branch": "main", + "revision": "2afa5040b567d1cdb6ac35e3a84ec764befe8711" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2" + }, + "historical_workflow": "linkerd2-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 13, + "native": { + "action": ".github/actions/linkerd2-docker-benchmark", + "benchmark_id": "linkerd2-v2", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "linkerd2-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 12, + "workflow_seconds": 15, + "total_seconds": 12, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 12 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": null + }, + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1230761384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 1230761384, + "unmeasured_tags": [], + "total_bytes": 1230761384, + "observations": [ + { + "requested_tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "tag": "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b542bd96-10d5-4849-96bc-e547137715ba", + "manifest_root_digest": "sha256:edb5fa1af4afd692a6cd8e570386819e7a508ccfa7b3d081483fd8440a631072", + "kv_total_size": 0, + "compressed_size": 1230761384, + "size": 1230761384 + } + ] + } + }, + "source": { + "repository": "linkerd/linkerd2", + "sha": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "linkerd2-v2-series-image-output-qualification-01-s1-r37104596700-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104596700", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170536", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104596700-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] +} diff --git a/results/mastodon/image-output-qualification-01/completions/37104599574.json b/results/mastodon/image-output-qualification-01/completions/37104599574.json new file mode 100644 index 000000000..a1bcfb0bd --- /dev/null +++ b/results/mastodon/image-output-qualification-01/completions/37104599574.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104599574", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2e2e3c52be0fbdb8c2ea7924416d0bc1530b9ee9b391a13b8f3fc03b0ab61e52" +} diff --git a/results/mastodon/image-output-qualification-01/evidence.md b/results/mastodon/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..359508519 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104599574 | 694784 | `8b6a772ff39503d0e9aac23a96e13f627e9ffbba1f4e84ece45598857559ee60` | 13 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/mastodon/image-output-qualification-01/interpretation.md b/results/mastodon/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..586a137a7 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# mastodon: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104599574) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both providers completed the declared cold and identical-source replay, including export and load. Logs and original phase records establish that the shared verifier inspected the selected local image after timing capture. This establishes image materialization, not runtime application correctness. Slower observations remain in the report. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/mastodon/image-output-qualification-01/report.json b/results/mastodon/image-output-qualification-01/report.json new file mode 100644 index 000000000..d9cba0286 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/report.json @@ -0,0 +1,975 @@ +{ + "schema_version": 1, + "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104599574", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2e2e3c52be0fbdb8c2ea7924416d0bc1530b9ee9b391a13b8f3fc03b0ab61e52" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 736, + "min": 736, + "max": 736 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 59, + "min": 59, + "max": 59 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 424, + "min": 424, + "max": 424 + }, + "storage": { + "median": 1130445165, + "min": 1130445165, + "max": 1130445165 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 38, + "min": 38, + "max": 38 + }, + "storage": { + "median": 1130445165, + "min": 1130445165, + "max": 1130445165 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 726, + "workflow_seconds": 742, + "total_seconds": 736, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 736 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170560", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 50, + "workflow_seconds": 64, + "total_seconds": 59, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 59 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170571", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 424, + "workflow_seconds": 431, + "total_seconds": 424, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 424 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + }, + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130445165, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130445165, + "unmeasured_tags": [], + "total_bytes": 1130445165, + "observations": [ + { + "requested_tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "cf2c7627-2c6c-4488-96c8-1950ff2bef65", + "manifest_root_digest": "sha256:7100d4bb076697242532af047831d4293f6a391848ad7c22240a5c388878ab7b", + "kv_total_size": 0, + "compressed_size": 1130445165, + "size": 1130445165 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170561", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 38, + "workflow_seconds": 46, + "total_seconds": 38, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 38 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130445165, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130445165, + "unmeasured_tags": [], + "total_bytes": 1130445165, + "observations": [ + { + "requested_tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "cf2c7627-2c6c-4488-96c8-1950ff2bef65", + "manifest_root_digest": "sha256:7100d4bb076697242532af047831d4293f6a391848ad7c22240a5c388878ab7b", + "kv_total_size": 0, + "compressed_size": 1130445165, + "size": 1130445165 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170570", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/image-output-qualification-01/report.md b/results/mastodon/image-output-qualification-01/report.md new file mode 100644 index 000000000..02165ca14 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/report.md @@ -0,0 +1,20 @@ +# mastodon: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 736 | 736–736 | unmeasured | 0 | +| warm | actions-cache | 1 | 59 | 59–59 | unmeasured | 0 | +| cold | boringcache | 1 | 424 | 424–424 | 1130445165 | 1 | +| warm | boringcache | 1 | 38 | 38–38 | 1130445165 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/mastodon/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/mastodon/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..eb7245913 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 726, + "workflow_seconds": 742, + "total_seconds": 736, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 736 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170560", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] +} diff --git a/results/mastodon/image-output-qualification-01/runs/1-actions-cache-warm.json b/results/mastodon/image-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..0835d7671 --- /dev/null +++ b/results/mastodon/image-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 50, + "workflow_seconds": 64, + "total_seconds": 59, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 59 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170571", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] +} diff --git a/results/mastodon/image-output-qualification-01/runs/1-boringcache-cold.json b/results/mastodon/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..f6046149a --- /dev/null +++ b/results/mastodon/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,244 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 424, + "workflow_seconds": 431, + "total_seconds": 424, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 424 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + }, + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130445165, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130445165, + "unmeasured_tags": [], + "total_bytes": 1130445165, + "observations": [ + { + "requested_tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "cf2c7627-2c6c-4488-96c8-1950ff2bef65", + "manifest_root_digest": "sha256:7100d4bb076697242532af047831d4293f6a391848ad7c22240a5c388878ab7b", + "kv_total_size": 0, + "compressed_size": 1130445165, + "size": 1130445165 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170561", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] +} diff --git a/results/mastodon/image-output-qualification-01/runs/1-boringcache-warm.json b/results/mastodon/image-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..88193e12e --- /dev/null +++ b/results/mastodon/image-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,244 @@ +{ + "schema_version": 1, + "benchmark": "mastodon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "server", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 38, + "workflow_seconds": 46, + "total_seconds": 38, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 38 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1130445165, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 1130445165, + "unmeasured_tags": [], + "total_bytes": 1130445165, + "observations": [ + { + "requested_tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "tag": "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "cf2c7627-2c6c-4488-96c8-1950ff2bef65", + "manifest_root_digest": "sha256:7100d4bb076697242532af047831d4293f6a391848ad7c22240a5c388878ab7b", + "kv_total_size": 0, + "compressed_size": 1130445165, + "size": 1130445165 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-series-image-output-qualification-01-s1-r37104599574-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104599574", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170570", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104599574-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] +} diff --git a/results/n8n/image-output-qualification-01/completions/37104598178.json b/results/n8n/image-output-qualification-01/completions/37104598178.json new file mode 100644 index 000000000..aa3f11c5e --- /dev/null +++ b/results/n8n/image-output-qualification-01/completions/37104598178.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104598178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5f0af1311b0f43733e4562504aed6f42dde1420f1c800abe59315d5f97a55214" +} diff --git a/results/n8n/image-output-qualification-01/evidence.md b/results/n8n/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..27bfbba61 --- /dev/null +++ b/results/n8n/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104598178 | 1748480 | `0bb9fc327d8fef9a8af1eb8bdcf8ec699508a46715bae610a28c12b0fc1dd49f` | 13 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/n8n/image-output-qualification-01/interpretation.md b/results/n8n/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..1993fbdb3 --- /dev/null +++ b/results/n8n/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# n8n: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104598178) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both providers completed the declared cold and identical-source replay, including export and load. Logs and original phase records establish that the shared verifier inspected the selected local image after timing capture. This establishes image materialization, not runtime application correctness. Slower observations remain in the report. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/n8n/image-output-qualification-01/report.json b/results/n8n/image-output-qualification-01/report.json new file mode 100644 index 000000000..f0d3dfff6 --- /dev/null +++ b/results/n8n/image-output-qualification-01/report.json @@ -0,0 +1,1079 @@ +{ + "schema_version": 1, + "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104598178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "5f0af1311b0f43733e4562504aed6f42dde1420f1c800abe59315d5f97a55214" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 446, + "min": 446, + "max": 446 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 150, + "min": 150, + "max": 150 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 319, + "min": 319, + "max": 319 + }, + "storage": { + "median": 483433384, + "min": 483433384, + "max": 483433384 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 162, + "min": 162, + "max": 162 + }, + "storage": { + "median": 483433384, + "min": 483433384, + "max": 483433384 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 436, + "workflow_seconds": 651, + "total_seconds": 446, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 446 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170539", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 139, + "workflow_seconds": 305, + "total_seconds": 150, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 150 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170547", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 319, + "workflow_seconds": 518, + "total_seconds": 319, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 319 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + }, + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483433384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483433384, + "unmeasured_tags": [], + "total_bytes": 483433384, + "observations": [ + { + "requested_tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "fa171c93-a2f1-4345-9568-235f8dc16ac7", + "manifest_root_digest": "sha256:248d63f97d956c0c98c50bb715f68e67dc6f22c0f7be5ee09bc2d3401dd513dd", + "kv_total_size": 0, + "compressed_size": 483433384, + "size": 483433384 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170540", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 162, + "workflow_seconds": 365, + "total_seconds": 162, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 162 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483433384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483433384, + "unmeasured_tags": [], + "total_bytes": 483433384, + "observations": [ + { + "requested_tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "fa171c93-a2f1-4345-9568-235f8dc16ac7", + "manifest_root_digest": "sha256:248d63f97d956c0c98c50bb715f68e67dc6f22c0f7be5ee09bc2d3401dd513dd", + "kv_total_size": 0, + "compressed_size": 483433384, + "size": 483433384 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170546", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/image-output-qualification-01/report.md b/results/n8n/image-output-qualification-01/report.md new file mode 100644 index 000000000..2bbb0a0f6 --- /dev/null +++ b/results/n8n/image-output-qualification-01/report.md @@ -0,0 +1,20 @@ +# n8n: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 446 | 446–446 | unmeasured | 0 | +| warm | actions-cache | 1 | 150 | 150–150 | unmeasured | 0 | +| cold | boringcache | 1 | 319 | 319–319 | 483433384 | 1 | +| warm | boringcache | 1 | 162 | 162–162 | 483433384 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/n8n/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/n8n/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..817dcd91a --- /dev/null +++ b/results/n8n/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 436, + "workflow_seconds": 651, + "total_seconds": 446, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 446 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170539", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] +} diff --git a/results/n8n/image-output-qualification-01/runs/1-actions-cache-warm.json b/results/n8n/image-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..250740cba --- /dev/null +++ b/results/n8n/image-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 139, + "workflow_seconds": 305, + "total_seconds": 150, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 150 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170547", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] +} diff --git a/results/n8n/image-output-qualification-01/runs/1-boringcache-cold.json b/results/n8n/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..d27e7b055 --- /dev/null +++ b/results/n8n/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 319, + "workflow_seconds": 518, + "total_seconds": 319, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 319 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + }, + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483433384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483433384, + "unmeasured_tags": [], + "total_bytes": 483433384, + "observations": [ + { + "requested_tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "fa171c93-a2f1-4345-9568-235f8dc16ac7", + "manifest_root_digest": "sha256:248d63f97d956c0c98c50bb715f68e67dc6f22c0f7be5ee09bc2d3401dd513dd", + "kv_total_size": 0, + "compressed_size": 483433384, + "size": 483433384 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170540", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] +} diff --git a/results/n8n/image-output-qualification-01/runs/1-boringcache-warm.json b/results/n8n/image-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..49ce00969 --- /dev/null +++ b/results/n8n/image-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "benchmark": "n8n-docker", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "docker", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 162, + "workflow_seconds": 365, + "total_seconds": 162, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 162 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 483433384, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 483433384, + "unmeasured_tags": [], + "total_bytes": 483433384, + "observations": [ + { + "requested_tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "tag": "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "fa171c93-a2f1-4345-9568-235f8dc16ac7", + "manifest_root_digest": "sha256:248d63f97d956c0c98c50bb715f68e67dc6f22c0f7be5ee09bc2d3401dd513dd", + "kv_total_size": 0, + "compressed_size": 483433384, + "size": 483433384 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-docker-series-image-output-qualification-01-s1-r37104598178-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104598178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170546", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104598178-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] +} diff --git a/results/posthog/image-output-combined-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/image-output-combined-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..4ade0a1ca --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "image-output-combined-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:34:21Z", + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "run_id": 37148377350, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350" +} diff --git a/results/posthog/image-output-qualification-01/completions/37104600856.json b/results/posthog/image-output-qualification-01/completions/37104600856.json new file mode 100644 index 000000000..84795af7e --- /dev/null +++ b/results/posthog/image-output-qualification-01/completions/37104600856.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104600856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog warm: failure", + "BoringCache posthog warm: failure" + ], + "evidence_manifest_sha256": "61aee72e30eb1e9f7e21dc0bf4c65e62b3d33de24057f5343593919036e6f320" +} diff --git a/results/posthog/image-output-qualification-01/evidence.md b/results/posthog/image-output-qualification-01/evidence.md new file mode 100644 index 000000000..9f183a6cb --- /dev/null +++ b/results/posthog/image-output-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37104600856 | 392704 | `6a38f37864533c9921020e3d994a21914adf337775f42b734e74fd894eebc7e2` | 9 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Failed execution retained; this archive does not qualify the workload. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/posthog/image-output-qualification-01/interpretation.md b/results/posthog/image-output-qualification-01/interpretation.md new file mode 100644 index 000000000..f5209ad7b --- /dev/null +++ b/results/posthog/image-output-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# posthog: image-output-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37104600856) executed its frozen request at `35e10981f5e27824d81b19c575a91691592905e9`. + +Both warm jobs failed during the legacy published-scope assertion. The two original cold records include loaded-image inspection inside build_seconds, contrary to the declared scope. The plan-bound methodology review rejects comparison. Corrected layers and combined execution use new plans and a new signed harness ref. + +The [generated report](report.md) retains 2 original phase records. Completion is `failed`; comparative use is `false`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/posthog/image-output-qualification-01/methodology-review.json b/results/posthog/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..491c1a0ea --- /dev/null +++ b/results/posthog/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", + "issues": [ + "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + ] +} diff --git a/results/posthog/image-output-qualification-01/report.json b/results/posthog/image-output-qualification-01/report.json new file mode 100644 index 000000000..7e8b8e81f --- /dev/null +++ b/results/posthog/image-output-qualification-01/report.json @@ -0,0 +1,527 @@ +{ + "schema_version": 1, + "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104600856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog warm: failure", + "BoringCache posthog warm: failure" + ], + "evidence_manifest_sha256": "61aee72e30eb1e9f7e21dc0bf4c65e62b3d33de24057f5343593919036e6f320" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104600856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog warm: failure", + "BoringCache posthog warm: failure" + ], + "evidence_manifest_sha256": "61aee72e30eb1e9f7e21dc0bf4c65e62b3d33de24057f5343593919036e6f320" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1579, + "min": 1579, + "max": 1579 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 813, + "min": 813, + "max": 813 + }, + "storage": { + "median": 6237669195, + "min": 6237669195, + "max": 6237669195 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 1572, + "workflow_seconds": 1594, + "total_seconds": 1579, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1579 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-image-output-qualification-01-s1-r37104600856-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104600856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170576", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104600856-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 813, + "workflow_seconds": 827, + "total_seconds": 813, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 813 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64", + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64" + }, + "tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237669195, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237669195, + "unmeasured_tags": [], + "total_bytes": 6237669195, + "observations": [ + { + "requested_tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64", + "tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "f74b368e-efea-4056-bd7f-187aeb41c702", + "manifest_root_digest": "sha256:8516b28a0de44c72b868915c3de0eb8ca70ec05fb57b701e43dc1da4895d4f70", + "kv_total_size": 0, + "compressed_size": 6237669195, + "size": 6237669195 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104600856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170577", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104600856-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/image-output-qualification-01/report.md b/results/posthog/image-output-qualification-01/report.md new file mode 100644 index 000000000..ce8f27746 --- /dev/null +++ b/results/posthog/image-output-qualification-01/report.md @@ -0,0 +1,29 @@ +# posthog: image-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: 2 declared observations missing; 0 failed. Publication requires review. + +Execution: unqualified; missing or failed job completion checks. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 1579 | 1579–1579 | unmeasured | 0 | +| cold | boringcache | 1 | 813 | 813–813 | 6237669195 | 1 | + +Methodology prevents a comparative claim: + +- Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution. +- Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series. + +Run 37104600856 failed completion checks: + +- Workflow concluded failure +- GitHub Actions posthog warm: failure +- BoringCache posthog warm: failure + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/image-output-qualification-01/runs/1-actions-cache-cold.json b/results/posthog/image-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..fe67119f1 --- /dev/null +++ b/results/posthog/image-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 1572, + "workflow_seconds": 1594, + "total_seconds": 1579, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1579 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-image-output-qualification-01-s1-r37104600856-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104600856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170576", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104600856-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + ] +} diff --git a/results/posthog/image-output-qualification-01/runs/1-boringcache-cold.json b/results/posthog/image-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..5db025590 --- /dev/null +++ b/results/posthog/image-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 813, + "workflow_seconds": 827, + "total_seconds": 813, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 813 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64", + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64" + }, + "tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237669195, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237669195, + "unmeasured_tags": [], + "total_bytes": 6237669195, + "observations": [ + { + "requested_tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64", + "tag": "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "f74b368e-efea-4056-bd7f-187aeb41c702", + "manifest_root_digest": "sha256:8516b28a0de44c72b868915c3de0eb8ca70ec05fb57b701e43dc1da4895d4f70", + "kv_total_size": 0, + "compressed_size": 6237669195, + "size": 6237669195 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-image-output-qualification-01-s1-r37104600856-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37104600856", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "image-output-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170577", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37104600856-1", + "series": { + "id": "image-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + ] +} diff --git a/results/posthog/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..2ce18dac1 --- /dev/null +++ b/results/posthog/image-output-qualification-02/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:34:19Z", + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "run_id": 37148375181, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181" +} diff --git a/results/storybook/build-reuse-qualification-01/completions/37105487087.json b/results/storybook/build-reuse-qualification-01/completions/37105487087.json new file mode 100644 index 000000000..fc8ce7c31 --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/completions/37105487087.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "storybook", + "series_id": "build-reuse-qualification-01", + "sample": 1, + "run_id": "37105487087", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "93bb5bb3faf15c087bb8fc3d967d8355c1246960603f6dc637057920c5b7c245" +} diff --git a/results/storybook/build-reuse-qualification-01/evidence.md b/results/storybook/build-reuse-qualification-01/evidence.md new file mode 100644 index 000000000..e207ce6c6 --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/evidence.md @@ -0,0 +1,16 @@ +# Preserved run evidence + +The [durable release](https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03) contains this scoped run export. + +| Run | Bundle bytes | SHA-256 | Verified files | Gaps | +| --- | ---: | --- | ---: | ---: | +| 37105487087 | 509440 | `512f07883a0dfbf61d823c68aca104b02abe640bc216f25f1b6d61805cf322bd` | 15 | 0 | + +The uploaded asset was independently downloaded, checked against its archive +digest, extracted, and verified against its file manifest and resource inventory. +This covers available attempts, jobs, logs, artifacts, commit metadata, and +workflow for this run. It does not audit the original repository. + +Scoped output and completion checks passed; one sample; publication unreviewed. +The [interpretation](interpretation.md) records the execution and claim limits. +Preservation does not approve publication or repository deletion. diff --git a/results/storybook/build-reuse-qualification-01/interpretation.md b/results/storybook/build-reuse-qualification-01/interpretation.md new file mode 100644 index 000000000..960a06d3e --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/interpretation.md @@ -0,0 +1,11 @@ +# storybook: build-reuse-qualification-01 + +The [original run](https://github.com/boringcache/benchmarks/actions/runs/37105487087) executed its frozen request at `b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e`. + +Dependency installation and sandbox creation occur outside the selected setup/build metric. Keep the earlier combined-scope screen diagnostic. This one-sample result retains BoringCache's slower cold and replay measurements. The check confirms the built benchmark output, not full application behavior. + +The [generated report](report.md) retains 4 original phase records. Completion is `success`; comparative use is `true`. Sample count is one; publication remains unreviewed. + +Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. + +Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. diff --git a/results/storybook/build-reuse-qualification-01/report.json b/results/storybook/build-reuse-qualification-01/report.json new file mode 100644 index 000000000..4f97c3223 --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/report.json @@ -0,0 +1,846 @@ +{ + "schema_version": 1, + "plan_sha256": "d463583c8fb964e5c3721430687e155f6579a1a423afa5508f758069e22942b5", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "storybook", + "series_id": "build-reuse-qualification-01", + "sample": 1, + "run_id": "37105487087", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37105487087", + "source_sha": "b3ab6324aeccebbb38cc7e2fc1c921ae4f0b399e", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "93bb5bb3faf15c087bb8fc3d967d8355c1246960603f6dc637057920c5b7c245" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 17, + "min": 17, + "max": 17 + }, + "storage": { + "median": 1226001, + "min": 1226001, + "max": 1226001 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 18, + "min": 18, + "max": 18 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 19, + "min": 19, + "max": 19 + }, + "storage": { + "median": 1212413, + "min": 1212413, + "max": 1212413 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "storybook", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 15, + "workflow_seconds": 269, + "total_seconds": 15, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 15 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "nx-storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-474a5399b5b2728b562d61ddc952ce4caa750fa0", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170624", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] + }, + { + "schema_version": 1, + "benchmark": "storybook", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 16, + "workflow_seconds": 270, + "total_seconds": 17, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 17 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 1226001, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "nx-storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-474a5399b5b2728b562d61ddc952ce4caa750fa0", + "complete": true, + "measured_bytes": 1226001, + "total_bytes": 1226001, + "observations": [ + { + "id": 8451338754, + "key": "nx-storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-474a5399b5b2728b562d61ddc952ce4caa750fa0", + "ref": "refs/heads/refs/tags/build-reuse-qualification-2026-10-03", + "version": "8448ebf87772b13034f4b4ab0e8486e13c9465998b6fef63efe165aff2d5a066", + "created_at": "2026-10-03T09:54:19.900749Z", + "last_accessed_at": "2026-10-03T09:55:26.115064Z", + "size_in_bytes": 1226001 + } + ] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170632", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] + }, + { + "schema_version": 1, + "benchmark": "storybook", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 13, + "workflow_seconds": 271, + "total_seconds": 18, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 18 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "status": "miss", + "kv_total_size": 0 + }, + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "archive", + "resolved_tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170623", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] + }, + { + "schema_version": 1, + "benchmark": "storybook", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4, + "build_seconds": 15, + "workflow_seconds": 274, + "total_seconds": 19, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 19 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1212413, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 1212413, + "unmeasured_tags": [], + "total_bytes": 1212413, + "observations": [ + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b797d1dd-e00c-48ba-b2ed-599eb4c03948", + "manifest_root_digest": "sha256:d6278319fff6d3e6cb0e6f0451b6cb13a3371467e462050f6becf01b654764cc", + "kv_total_size": 0, + "compressed_size": 12657, + "size": 12657 + }, + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b372e2c4-c217-4c69-8317-622d509906fa", + "manifest_root_digest": "sha256:3edd9024864b909d313164200772fbabcd2d8debc0b5e54238d90a283e4be42d", + "kv_total_size": 0, + "compressed_size": 1199756, + "size": 1199756 + } + ] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "archive", + "resolved_tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170631", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] + } + ], + "exclusions": [] +} diff --git a/results/storybook/build-reuse-qualification-01/report.md b/results/storybook/build-reuse-qualification-01/report.md new file mode 100644 index 000000000..b3fba9f20 --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/report.md @@ -0,0 +1,20 @@ +# storybook: build-reuse-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 15 | 15–15 | unmeasured | 0 | +| warm | actions-cache | 1 | 17 | 17–17 | 1226001 | 1 | +| cold | boringcache | 1 | 18 | 18–18 | unmeasured | 0 | +| warm | boringcache | 1 | 19 | 19–19 | 1212413 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/storybook/build-reuse-qualification-01/runs/1-actions-cache-cold.json b/results/storybook/build-reuse-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..2d0fd9ace --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,162 @@ +{ + "schema_version": 1, + "benchmark": "storybook", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 15, + "workflow_seconds": 269, + "total_seconds": 15, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 15 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "nx-storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-474a5399b5b2728b562d61ddc952ce4caa750fa0", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170624", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] +} diff --git a/results/storybook/build-reuse-qualification-01/runs/1-actions-cache-warm.json b/results/storybook/build-reuse-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..7698125ad --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,172 @@ +{ + "schema_version": 1, + "benchmark": "storybook", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 16, + "workflow_seconds": 270, + "total_seconds": 17, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 17 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 1226001, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "nx-storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-474a5399b5b2728b562d61ddc952ce4caa750fa0", + "complete": true, + "measured_bytes": 1226001, + "total_bytes": 1226001, + "observations": [ + { + "id": 8451338754, + "key": "nx-storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-474a5399b5b2728b562d61ddc952ce4caa750fa0", + "ref": "refs/heads/refs/tags/build-reuse-qualification-2026-10-03", + "version": "8448ebf87772b13034f4b4ab0e8486e13c9465998b6fef63efe165aff2d5a066", + "created_at": "2026-10-03T09:54:19.900749Z", + "last_accessed_at": "2026-10-03T09:55:26.115064Z", + "size_in_bytes": 1226001 + } + ] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170632", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] +} diff --git a/results/storybook/build-reuse-qualification-01/runs/1-boringcache-cold.json b/results/storybook/build-reuse-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..3fa46db4f --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "benchmark": "storybook", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 13, + "workflow_seconds": 271, + "total_seconds": 18, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 18 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "status": "miss", + "kv_total_size": 0 + }, + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "archive", + "resolved_tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170623", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] +} diff --git a/results/storybook/build-reuse-qualification-01/runs/1-boringcache-warm.json b/results/storybook/build-reuse-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..8dc7404ec --- /dev/null +++ b/results/storybook/build-reuse-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,214 @@ +{ + "schema_version": 1, + "benchmark": "storybook", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "archive", + "adapter": "archive", + "case": { + "schema_version": 1, + "case_id": "storybook", + "workspace": "boringcache/benchmarks", + "definition_sha256": "25acb76c77bd69ce87f69a875238a0a71ad6b1a8acda4cac8ef51092bd3dfc28", + "source": { + "repository": "storybookjs/storybook", + "pins": [ + { + "path": "upstream", + "revision": "474a5399b5b2728b562d61ddc952ce4caa750fa0", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-storybook", + "branch": "main", + "revision": "a21c17fbfbb0f903ee7664e0b59d17726d8c5301" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "storybook" + }, + "historical_workflow": "storybook-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 26, + "native": { + "action": ".github/actions/storybook-nx-benchmark", + "benchmark_id": "storybook", + "rolling_inputs": { + "node_version": "22.22.3" + }, + "environment": {}, + "fresh_inputs": { + "node_version": "22.22.3" + } + }, + "cache_tags": { + "storybook-nx-cache-local": "{scope}-nx-cache", + "storybook-nx-workspace-data-local": "{scope}-nx-workspace-data" + } + } + }, + "timing": { + "restore_or_setup_seconds": 4, + "build_seconds": 15, + "workflow_seconds": 274, + "total_seconds": 19, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Storybook sandbox build plus measured cache restore/setup; dependency installation and sandbox creation are excluded; output checks and unmeasured post-job save are reported separately", + "build_and_reuse_seconds": 19 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache", + "workspace": "boringcache/benchmarks", + "storage_bytes": 1212413, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 1212413, + "unmeasured_tags": [], + "total_bytes": 1212413, + "observations": [ + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b797d1dd-e00c-48ba-b2ed-599eb4c03948", + "manifest_root_digest": "sha256:d6278319fff6d3e6cb0e6f0451b6cb13a3371467e462050f6becf01b654764cc", + "kv_total_size": 0, + "compressed_size": 12657, + "size": 12657 + }, + { + "requested_tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "tag": "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "b372e2c4-c217-4c69-8317-622d509906fa", + "manifest_root_digest": "sha256:3edd9024864b909d313164200772fbabcd2d8debc0b5e54238d90a283e4be42d", + "kv_total_size": 0, + "compressed_size": 1199756, + "size": 1199756 + } + ] + } + }, + "source": { + "repository": "storybookjs/storybook", + "sha": "474a5399b5b2728b562d61ddc952ce4caa750fa0" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "archive", + "resolved_tags": [ + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-cache-ubuntu-24-x86_64", + "storybook-series-build-reuse-qualification-01-s1-r37105487087-a1-nx-workspace-data-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37105487087", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "build-reuse-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000170631", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37105487087-1", + "series": { + "id": "build-reuse-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37105487087" + ] +} From b759879316279afa06a7e8166c905429547adb34 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 20:45:04 +0100 Subject: [PATCH 11/17] Record scoped qualification outcomes and remaining active variants --- data/latest/series.json | 201 ++++++++++++++++++ migration/repositories.json | 114 +++++++++- .../series.json | 46 ++++ .../series.json | 46 ++++ .../series.json | 46 ++++ 5 files changed, 442 insertions(+), 11 deletions(-) create mode 100644 results/mastodon/streaming-output-qualification-01/series.json create mode 100644 results/n8n/distroless-output-qualification-01/series.json create mode 100644 results/n8n/runners-output-qualification-01/series.json diff --git a/data/latest/series.json b/data/latest/series.json index cf34e5630..2829baea3 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -3461,6 +3461,140 @@ ], "runs_without_archives": [] }, + { + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", + "plan_path": "results/mastodon/streaming-output-qualification-01/series.json", + "lane": "fresh", + "variant": "streaming", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", + "plan_path": "results/n8n/distroless-output-qualification-01/series.json", + "lane": "fresh", + "variant": "distroless", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "n8n", "series_id": "image-output-qualification-01", @@ -3611,6 +3745,73 @@ ], "runs_without_archives": [] }, + { + "case_id": "n8n", + "series_id": "runners-output-qualification-01", + "plan_path": "results/n8n/runners-output-qualification-01/series.json", + "lane": "fresh", + "variant": "runners", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "n8n", "series_id": "shared-turbo-qualification-01", diff --git a/migration/repositories.json b/migration/repositories.json index 20a4843ab..856451153 100644 --- a/migration/repositories.json +++ b/migration/repositories.json @@ -13,7 +13,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "Initial screen cancelled after missing output-image verification was found; completion and zero-observation report retained; a corrected definition requires a new series", + "consolidated_execution": "Selected one-sample cold/replay image materialization and completion checks passed with a verified scoped archive; publication, other variants, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -39,6 +39,15 @@ 37085333990 ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03" + }, + "image_output_screening": { + "case_id": "chroma", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104573058 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" } }, { @@ -57,7 +66,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "verified direct single-provider Cargo product and compiler-only cold/adjacent-source proofs; native cache errors retained; Cargo storage total unqualified; rolling and full release caller paths remain unverified", + "consolidated_execution": "Both fresh parent/restore-only proofs and publication-capable adjacent-source runs completed for Cargo-product and compiler-only profiles; native errors retained; Cargo storage total unmeasured; generic verified lineage and full caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -131,6 +140,18 @@ "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", "https://github.com/boringcache/benchmarks/releases/tag/evidence-active-completion-2026-10-03" ] + }, + "rolling_seed_advance": { + "series_ids": [ + "rolling-cargo-product-seed-advance-01", + "rolling-compiler-only-seed-advance-01" + ], + "run_ids": [ + 37105483797, + 37105485279 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "scope": "Retained parent cohorts, verified binaries, publication-capable operations, and preserved completion; single-provider correctness evidence; native errors, incomplete selected-tag storage, and unverified generic lineage remain explicit" } }, { @@ -241,7 +262,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "incomplete; only recent run metadata inventoried", - "consolidated_execution": "Initial screen cancelled after missing output-image verification was found; completion and zero-observation report retained; a corrected definition requires a new series", + "consolidated_execution": "Selected one-sample cold/replay image materialization and completion checks passed with a verified scoped archive; publication, other variants, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -257,6 +278,15 @@ 37085335883 ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03" + }, + "image_output_screening": { + "case_id": "duckgres", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104593175 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" } }, { @@ -304,7 +334,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "Initial screen cancelled after missing output-image verification was found; completion and zero-observation report retained; a corrected definition requires a new series", + "consolidated_execution": "Both loaded-image cold builds failed on the attestation manifest list; no verified measurements; a separate corrected loaded-image series is requested", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -331,6 +361,15 @@ 37085337504 ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03" + }, + "image_output_screening": { + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104595025 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Original failed request and any phase records retained; corrected execution requires a separate frozen series" } }, { @@ -394,7 +433,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "unverified", + "consolidated_execution": "Selected one-sample cold/replay image materialization and completion checks passed with a verified scoped archive; publication, other variants, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -413,6 +452,15 @@ "complete": 2 }, "scope": "Website-linked run resources only; non-default branches, other run history, callers, and external references remain unaudited" + }, + "image_output_screening": { + "case_id": "immich", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104602471 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" } }, { @@ -427,7 +475,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "Initial screen cancelled after missing output-image verification was found; completion and zero-observation report retained; a corrected definition requires a new series", + "consolidated_execution": "Selected one-sample cold/replay image materialization and completion checks passed with a verified scoped archive; publication, other variants, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -453,6 +501,15 @@ 37085339156 ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03" + }, + "image_output_screening": { + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104596700 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" } }, { @@ -468,7 +525,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "unverified", + "consolidated_execution": "Selected one-sample cold/replay image materialization and completion checks passed with a verified scoped archive; publication, other variants, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -488,6 +545,15 @@ "complete": 3 }, "scope": "Website-linked run resources only; non-default branches, other run history, callers, and external references remain unaudited" + }, + "image_output_screening": { + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104599574 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" } }, { @@ -501,7 +567,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "unverified", + "consolidated_execution": "Turbo and Docker selected one-sample cold/replay output and completion checks passed with verified archives; runners, distroless, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -520,6 +586,15 @@ "complete": 2 }, "scope": "Website-linked run resources only; non-default branches, other run history, callers, and external references remain unaudited" + }, + "image_output_screening": { + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104598178 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" } }, { @@ -627,7 +702,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "unverified", + "consolidated_execution": "Both warm jobs failed during scope selection; original cold timings include inspection contrary to the declared scope; methodology review rejects comparison; corrected layers and combined series are requested", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -651,6 +726,15 @@ "complete": 7 }, "scope": "Website-linked run resources only; non-default branches, other run history, callers, and external references remain unaudited" + }, + "image_output_screening": { + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "run_ids": [ + 37104600856 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "scope": "Original failed request and any phase records retained; corrected execution requires a separate frozen series" } }, { @@ -746,7 +830,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "One-sample fresh output and completion checks passed; setup timing includes dependency installation and sandbox creation, so the methodology review rejects comparative use; corrected boundary requires a new series", + "consolidated_execution": "Corrected one-sample cold/replay output and completion checks passed with dependency installation and sandbox creation outside the metric; the original combined-scope screen remains diagnostic; publication and caller cutover unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -773,6 +857,14 @@ ], "scope": "Diagnostic combined measurement; all original observations retained; no build-and-reuse comparison", "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-active-completion-2026-10-03" + }, + "build_reuse_screening": { + "series_id": "build-reuse-qualification-01", + "run_ids": [ + 37105487087 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-build-reuse-2026-10-03", + "scope": "Corrected timing boundary; one sample; slower BoringCache measurements retained; missing storage unmeasured; publication unreviewed" } }, { @@ -789,7 +881,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "unverified", + "consolidated_execution": "Combined cold/changed-source one-sample proof passed output and preserved completion checks with a verified archive; other layers, rolling behavior, and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ diff --git a/results/mastodon/streaming-output-qualification-01/series.json b/results/mastodon/streaming-output-qualification-01/series.json new file mode 100644 index 000000000..1d3f4b917 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", + "created_at": "2026-10-03T19:44:44Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "variant": "streaming", + "workflow_inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "streaming" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af" +} diff --git a/results/n8n/distroless-output-qualification-01/series.json b/results/n8n/distroless-output-qualification-01/series.json new file mode 100644 index 000000000..873643b7d --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", + "created_at": "2026-10-03T19:44:44Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "variant": "distroless", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "distroless" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55" +} diff --git a/results/n8n/runners-output-qualification-01/series.json b/results/n8n/runners-output-qualification-01/series.json new file mode 100644 index 000000000..41cfd35fe --- /dev/null +++ b/results/n8n/runners-output-qualification-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "runners-output-qualification-01", + "created_at": "2026-10-03T19:44:44Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "variant": "runners", + "workflow_inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "runners" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e" +} From 3c0aeb8eed7634a043ed3475ab5bee1966e93f95 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sat, 3 Oct 2026 20:46:33 +0100 Subject: [PATCH 12/17] Retain requests for remaining fresh variants --- data/latest/series.json | 81 +++++++++-- ...tive-variant-qualification-2026-10-03.json | 128 ++++++++++++++++++ .../1-native-fresh-benchmark.yml.json | 39 ++++++ .../1-native-fresh-benchmark.yml.json | 40 ++++++ .../1-native-fresh-benchmark.yml.json | 40 ++++++ 5 files changed, 319 insertions(+), 9 deletions(-) create mode 100644 migration/active-variant-qualification-2026-10-03.json create mode 100644 results/mastodon/streaming-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/n8n/distroless-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/n8n/runners-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json diff --git a/data/latest/series.json b/data/latest/series.json index 2829baea3..c4860b050 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -3492,7 +3492,7 @@ }, "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -3520,13 +3520,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "streaming", + "series_id": "streaming-output-qualification-01", + "sample": "1" + }, + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:45:29Z", + "run_id": 37149053291, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37149053291" + ] }, { "case_id": "n8n", @@ -3559,7 +3580,7 @@ }, "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -3587,13 +3608,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "distroless", + "series_id": "distroless-output-qualification-01", + "sample": "1" + }, + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:45:33Z", + "run_id": 37149056843, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37149056843" + ] }, { "case_id": "n8n", @@ -3776,7 +3818,7 @@ }, "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -3804,13 +3846,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "runners", + "series_id": "runners-output-qualification-01", + "sample": "1" + }, + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:45:31Z", + "run_id": 37149055130, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37149055130" + ] }, { "case_id": "n8n", diff --git a/migration/active-variant-qualification-2026-10-03.json b/migration/active-variant-qualification-2026-10-03.json new file mode 100644 index 000000000..defe67498 --- /dev/null +++ b/migration/active-variant-qualification-2026-10-03.json @@ -0,0 +1,128 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "scope": "Three predeclared one-sample screens for remaining selected fresh workload variants; request acceptance does not establish execution or qualify publication.", + "publication": "unreviewed", + "dispatches": [ + { + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "streaming", + "series_id": "streaming-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:45:29Z", + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "run_id": 37149053291, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + }, + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "runners", + "series_id": "runners-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:45:31Z", + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "run_id": 37149055130, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + }, + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "distroless", + "series_id": "distroless-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:45:33Z", + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "run_id": 37149056843, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + } + ] +} diff --git a/results/mastodon/streaming-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/mastodon/streaming-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..570417a7f --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,39 @@ +{ + "case_id": "mastodon", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "mastodon", + "cli_version": "v1.33.0", + "variant": "streaming", + "series_id": "streaming-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:45:29Z", + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "run_id": 37149053291, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291" +} diff --git a/results/n8n/distroless-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/n8n/distroless-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..17247dc4c --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "distroless", + "series_id": "distroless-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:45:33Z", + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "run_id": 37149056843, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843" +} diff --git a/results/n8n/runners-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/n8n/runners-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..98e786164 --- /dev/null +++ b/results/n8n/runners-output-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,40 @@ +{ + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "runners", + "series_id": "runners-output-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "schema_version": 1, + "requested_at": "2026-10-03T19:45:31Z", + "ref": "active-variant-qualification-2026-10-03", + "state": "requested", + "run_id": 37149055130, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130" +} From b5f0bd5c75967d3ea6600b8c12117d0ef73a38e5 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sun, 4 Oct 2026 14:07:47 +0100 Subject: [PATCH 13/17] Share Docker lifecycle and preserve final benchmark evidence --- .github/actions/boringcache/action.yml | 10 + .github/actions/docker-benchmark/action.yml | 119 + .github/actions/prepare-case/action.yml | 3 + .../retain-product-evidence/action.yml | 10 + .../actions/retain-product-evidence/index.cjs | 48 + .../retain-product-evidence/package-lock.json | 2527 +++++++++++++++++ .../retain-product-evidence/package.json | 8 + .github/workflows/guardrails.yml | 2 + .gitignore | 1 + bin/bench | 11 +- .../chroma-docker-benchmark/action.yml | 102 +- .../duckgres-docker-benchmark/action.yml | 97 +- .../actions/hugo-go-benchmark/action.yml | 25 +- .../actions/hugo-docker-benchmark/action.yml | 95 +- .../immich-base-images-benchmark/action.yml | 67 +- .../immich-docker-benchmark/action.yml | 84 +- .../linkerd2-docker-benchmark/action.yml | 93 +- .../mastodon-docker-benchmark/action.yml | 86 +- .../actions/n8n-docker-benchmark/action.yml | 94 +- .../actions/n8n-turbo-benchmark/action.yml | 19 +- .../opentelemetry-gradle-benchmark/action.yml | 25 +- .../posthog-docker-benchmark/action.yml | 99 +- .../qdrant-docker-benchmark/action.yml | 91 +- .../actions/spring-maven-benchmark/action.yml | 25 +- .../actions/storybook-nx-benchmark/action.yml | 27 +- data/latest/series.json | 857 ++++-- docs/migration.md | 23 + docs/process.md | 31 +- migration/evidence.json | 96 + .../interpretation.md | 2 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 7 +- .../image-output-qualification-01/report.md | 5 + .../interpretation.md | 2 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 7 +- .../image-output-qualification-01/report.md | 5 + .../completions/37148378881.json | 13 + .../interpretation.md | 7 + .../methodology-review.json | 11 + .../image-output-qualification-02/report.json | 836 ++++++ .../image-output-qualification-02/report.md | 25 + .../runs/1-actions-cache-cold.json | 163 ++ .../runs/1-actions-cache-warm.json | 163 ++ .../runs/1-boringcache-cold.json | 208 ++ .../runs/1-boringcache-warm.json | 208 ++ .../interpretation.md | 2 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 7 +- .../image-output-qualification-01/report.md | 5 + .../interpretation.md | 2 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 7 +- .../image-output-qualification-01/report.md | 5 + .../interpretation.md | 2 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 7 +- .../image-output-qualification-01/report.md | 5 + .../completions/37149053291.json | 13 + .../interpretation.md | 7 + .../methodology-review.json | 11 + .../report.json | 978 +++++++ .../report.md | 25 + .../runs/1-actions-cache-cold.json | 198 ++ .../runs/1-actions-cache-warm.json | 198 ++ .../runs/1-boringcache-cold.json | 244 ++ .../runs/1-boringcache-warm.json | 244 ++ .../completions/37149056843.json | 13 + .../interpretation.md | 7 + .../methodology-review.json | 11 + .../report.json | 1082 +++++++ .../report.md | 25 + .../runs/1-actions-cache-cold.json | 224 ++ .../runs/1-actions-cache-warm.json | 224 ++ .../runs/1-boringcache-cold.json | 270 ++ .../runs/1-boringcache-warm.json | 270 ++ .../interpretation.md | 2 + .../methodology-review.json | 11 + .../image-output-qualification-01/report.json | 7 +- .../image-output-qualification-01/report.md | 5 + .../completions/37149055130.json | 13 + .../interpretation.md | 7 + .../methodology-review.json | 11 + .../report.json | 1082 +++++++ .../runners-output-qualification-01/report.md | 25 + .../runs/1-actions-cache-cold.json | 224 ++ .../runs/1-actions-cache-warm.json | 224 ++ .../runs/1-boringcache-cold.json | 270 ++ .../runs/1-boringcache-warm.json | 270 ++ .../completions/37148377350.json | 13 + .../interpretation.md | 7 + .../methodology-review.json | 11 + .../report.json | 950 +++++++ .../report.md | 25 + .../runs/1-actions-cache-cold.json | 191 ++ .../runs/1-actions-cache-warm.json | 191 ++ .../runs/1-boringcache-cold.json | 237 ++ .../runs/1-boringcache-warm.json | 237 ++ .../interpretation.md | 2 + .../methodology-review.json | 4 +- .../image-output-qualification-01/report.json | 4 +- .../image-output-qualification-01/report.md | 2 + .../completions/37148375181.json | 13 + .../interpretation.md | 7 + .../methodology-review.json | 11 + .../image-output-qualification-02/report.json | 950 +++++++ .../image-output-qualification-02/report.md | 25 + .../runs/1-actions-cache-cold.json | 191 ++ .../runs/1-actions-cache-warm.json | 191 ++ .../runs/1-boringcache-cold.json | 237 ++ .../runs/1-boringcache-warm.json | 237 ++ .../migration-screening-01/interpretation.md | 2 + .../methodology-review.json | 11 + .../qdrant/migration-screening-01/report.json | 7 +- .../qdrant/migration-screening-01/report.md | 5 + scripts/benchmark-cases.rb | 25 +- scripts/benchmark-phase.rb | 55 + scripts/collect-series.rb | 64 + scripts/nightly-canaries.rb | 12 +- suites/published.json | 60 +- test/benchmark_case_scripts_test.rb | 2 +- test/benchmark_cases_test.rb | 14 + test/benchmark_phase_test.rb | 40 + test/benchmark_series_test.rb | 49 + test/native_case_test.rb | 17 +- test/nightly_canaries_test.rb | 22 + test/product_evidence_test.cjs | 38 + 127 files changed, 15771 insertions(+), 1117 deletions(-) create mode 100644 .github/actions/docker-benchmark/action.yml create mode 100644 .github/actions/retain-product-evidence/action.yml create mode 100644 .github/actions/retain-product-evidence/index.cjs create mode 100644 .github/actions/retain-product-evidence/package-lock.json create mode 100644 .github/actions/retain-product-evidence/package.json create mode 100644 results/chroma/image-output-qualification-01/methodology-review.json create mode 100644 results/duckgres/image-output-qualification-01/methodology-review.json create mode 100644 results/hugo/image-output-qualification-02/completions/37148378881.json create mode 100644 results/hugo/image-output-qualification-02/interpretation.md create mode 100644 results/hugo/image-output-qualification-02/methodology-review.json create mode 100644 results/hugo/image-output-qualification-02/report.json create mode 100644 results/hugo/image-output-qualification-02/report.md create mode 100644 results/hugo/image-output-qualification-02/runs/1-actions-cache-cold.json create mode 100644 results/hugo/image-output-qualification-02/runs/1-actions-cache-warm.json create mode 100644 results/hugo/image-output-qualification-02/runs/1-boringcache-cold.json create mode 100644 results/hugo/image-output-qualification-02/runs/1-boringcache-warm.json create mode 100644 results/immich/image-output-qualification-01/methodology-review.json create mode 100644 results/linkerd2/image-output-qualification-01/methodology-review.json create mode 100644 results/mastodon/image-output-qualification-01/methodology-review.json create mode 100644 results/mastodon/streaming-output-qualification-01/completions/37149053291.json create mode 100644 results/mastodon/streaming-output-qualification-01/interpretation.md create mode 100644 results/mastodon/streaming-output-qualification-01/methodology-review.json create mode 100644 results/mastodon/streaming-output-qualification-01/report.json create mode 100644 results/mastodon/streaming-output-qualification-01/report.md create mode 100644 results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/mastodon/streaming-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/mastodon/streaming-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/n8n/distroless-output-qualification-01/completions/37149056843.json create mode 100644 results/n8n/distroless-output-qualification-01/interpretation.md create mode 100644 results/n8n/distroless-output-qualification-01/methodology-review.json create mode 100644 results/n8n/distroless-output-qualification-01/report.json create mode 100644 results/n8n/distroless-output-qualification-01/report.md create mode 100644 results/n8n/distroless-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/n8n/distroless-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/n8n/distroless-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/n8n/distroless-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/n8n/image-output-qualification-01/methodology-review.json create mode 100644 results/n8n/runners-output-qualification-01/completions/37149055130.json create mode 100644 results/n8n/runners-output-qualification-01/interpretation.md create mode 100644 results/n8n/runners-output-qualification-01/methodology-review.json create mode 100644 results/n8n/runners-output-qualification-01/report.json create mode 100644 results/n8n/runners-output-qualification-01/report.md create mode 100644 results/n8n/runners-output-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/n8n/runners-output-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/n8n/runners-output-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/n8n/runners-output-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/posthog/image-output-combined-qualification-01/completions/37148377350.json create mode 100644 results/posthog/image-output-combined-qualification-01/interpretation.md create mode 100644 results/posthog/image-output-combined-qualification-01/methodology-review.json create mode 100644 results/posthog/image-output-combined-qualification-01/report.json create mode 100644 results/posthog/image-output-combined-qualification-01/report.md create mode 100644 results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/posthog/image-output-combined-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/posthog/image-output-combined-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/posthog/image-output-qualification-02/completions/37148375181.json create mode 100644 results/posthog/image-output-qualification-02/interpretation.md create mode 100644 results/posthog/image-output-qualification-02/methodology-review.json create mode 100644 results/posthog/image-output-qualification-02/report.json create mode 100644 results/posthog/image-output-qualification-02/report.md create mode 100644 results/posthog/image-output-qualification-02/runs/1-actions-cache-cold.json create mode 100644 results/posthog/image-output-qualification-02/runs/1-actions-cache-warm.json create mode 100644 results/posthog/image-output-qualification-02/runs/1-boringcache-cold.json create mode 100644 results/posthog/image-output-qualification-02/runs/1-boringcache-warm.json create mode 100644 results/qdrant/migration-screening-01/methodology-review.json create mode 100644 scripts/benchmark-phase.rb create mode 100644 scripts/collect-series.rb create mode 100644 test/benchmark_phase_test.rb create mode 100644 test/product_evidence_test.cjs diff --git a/.github/actions/boringcache/action.yml b/.github/actions/boringcache/action.yml index 3a243406c..d2bd8c2a7 100644 --- a/.github/actions/boringcache/action.yml +++ b/.github/actions/boringcache/action.yml @@ -35,6 +35,9 @@ outputs: runs: using: composite steps: + # Register first so retention runs after the product's post step. + - id: retention + uses: ./.github/actions/retain-product-evidence - id: provider uses: boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79 # v1.33.0 with: @@ -47,3 +50,10 @@ runs: diagnostics: ${{ inputs.diagnostics }} fail-on-cache-error: true fail-on-cache-miss: ${{ inputs.fail-on-cache-miss == 'true' }} + - name: Register the original product evidence + if: always() + shell: bash + env: + EVIDENCE_PATH: ${{ steps.provider.outputs.evidence-path }} + REGISTRY_PATH: ${{ steps.retention.outputs.registry-path }} + run: printf '%s\n' "$EVIDENCE_PATH" > "$REGISTRY_PATH" diff --git a/.github/actions/docker-benchmark/action.yml b/.github/actions/docker-benchmark/action.yml new file mode 100644 index 000000000..aa6531aaa --- /dev/null +++ b/.github/actions/docker-benchmark/action.yml @@ -0,0 +1,119 @@ +name: Build with the declared Docker cache provider +description: Measures the same cache publication policy and image output for both providers. +inputs: + strategy: + required: true + phase: + required: true + publish-on-warm: + default: 'false' + cache-scope: + required: true + cli-version: + default: '' + cli-platform: + default: '' + buildkit-image: + default: '' + context: + required: true + file: + required: true + platforms: + default: '' + target: + default: '' + tags: + default: '' + push: + default: 'false' + load: + default: 'false' + build-args: + default: '' + provenance: + default: '' + sbom: + default: 'false' + secrets: + default: '' + no-cache: + default: 'false' +outputs: + setup-seconds: + value: ${{ steps.build_timer.outputs.setup_seconds }} + description: Provider setup duration. + build-seconds: + value: ${{ steps.build_timing.outputs.build_seconds }} + description: Build, image output and declared cache publication duration. + cache-hit: + value: ${{ steps.provider.outputs.cache-hit }} + description: Product-reported cache outcome, when measured. + evidence-path: + value: ${{ steps.provider.outputs.evidence-path }} + description: Original product evidence file. +runs: + using: composite + steps: + - name: Resolve the shared cache publication policy + id: policy + shell: bash + env: + PHASE: ${{ inputs.phase }} + PUBLISH_ON_WARM: ${{ inputs.publish-on-warm }} + run: ruby ./scripts/benchmark-phase.rb policy + - name: Start the provider setup timer + id: setup_timer + shell: bash + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Set up the Actions Cache builder + if: inputs.strategy == 'actions-cache' + uses: ./.github/actions/setup-docker + - name: Start the build timer + id: build_timer + shell: bash + env: + SETUP_STARTED_AT: ${{ steps.setup_timer.outputs.started_at }} + run: | + now="$(date +%s)" + echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" + echo "started_at=$now" >> "$GITHUB_OUTPUT" + - name: Build with Actions Cache + if: inputs.strategy == 'actions-cache' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + env: + DOCKER_BUILD_SUMMARY: 'false' + DOCKER_BUILD_RECORD_UPLOAD: 'false' + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + target: ${{ inputs.target }} + tags: ${{ inputs.tags }} + push: ${{ inputs.push == 'true' }} + load: ${{ inputs.load == 'true' }} + build-args: ${{ inputs.build-args }} + provenance: ${{ inputs.provenance }} + sbom: ${{ inputs.sbom == 'true' }} + secrets: ${{ inputs.secrets }} + no-cache: ${{ inputs.no-cache == 'true' }} + cache-from: type=gha,scope=${{ inputs.cache-scope }} + cache-to: ${{ steps.policy.outputs.publish-cache == 'true' && format('type=gha,scope={0},mode=max', inputs.cache-scope) || '' }} + - name: Build with BoringCache + id: provider + if: inputs.strategy == 'boringcache' + uses: ./.github/actions/boringcache + env: + BORINGCACHE_MANAGED_BUILDKIT_IMAGE: ${{ inputs.buildkit-image }} + with: + mode: docker + cli-version: ${{ inputs.cli-version }} + cli-platform: ${{ inputs.cli-platform }} + trust-policy: ${{ steps.policy.outputs.trust-policy }} + fail-on-cache-miss: ${{ steps.policy.outputs.require-hit }} + - name: Capture the build timing + id: build_timing + shell: bash + env: + BUILD_STARTED_AT: ${{ steps.build_timer.outputs.started_at }} + run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/prepare-case/action.yml b/.github/actions/prepare-case/action.yml index d28e8bf6c..eaabe2209 100644 --- a/.github/actions/prepare-case/action.yml +++ b/.github/actions/prepare-case/action.yml @@ -41,3 +41,6 @@ runs: fi fi ruby .harness/bin/bench prepare "$BENCHMARK_CASE_ID" --directory "$GITHUB_WORKSPACE" "${args[@]}" + - name: Install the GitHub artifact client + shell: bash + run: npm ci --ignore-scripts --no-audit --no-fund --prefix .github/actions/retain-product-evidence diff --git a/.github/actions/retain-product-evidence/action.yml b/.github/actions/retain-product-evidence/action.yml new file mode 100644 index 000000000..8a62dd186 --- /dev/null +++ b/.github/actions/retain-product-evidence/action.yml @@ -0,0 +1,10 @@ +name: Retain product lifecycle evidence +description: Uploads the original product evidence after its post step completes. +outputs: + registry-path: + description: File in which the wrapper records the public evidence-path output. +runs: + using: node24 + main: index.cjs + post: index.cjs + post-if: always() diff --git a/.github/actions/retain-product-evidence/index.cjs b/.github/actions/retain-product-evidence/index.cjs new file mode 100644 index 000000000..01e1c6f3d --- /dev/null +++ b/.github/actions/retain-product-evidence/index.cjs @@ -0,0 +1,48 @@ +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +function register(core) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'benchmark-evidence-')); + const registry = path.join(directory, 'evidence-path'); + fs.writeFileSync(registry, '', { mode: 0o600 }); + core.saveState('evidence-directory', directory); + core.setOutput('registry-path', registry); +} + +async function retain(core, artifact) { + const directory = core.getState('evidence-directory'); + const source = fs.readFileSync(path.join(directory, 'evidence-path'), 'utf8').trim(); + if (!source) { + throw new Error('The product did not return an evidence path.'); + } + const name = path.basename(source); + if (!/^boringcache-one-evidence-[a-f0-9]{64}\.json$/.test(name)) { + throw new Error('The product returned an unexpected evidence filename.'); + } + const stat = fs.lstatSync(source); + if (!stat.isFile() || stat.size > 8 * 1024 * 1024) { + throw new Error('Product evidence must be a regular JSON file no larger than 8 MiB.'); + } + const content = fs.readFileSync(source); + const evidence = JSON.parse(content); + if (evidence.schema_version !== 'boringcache_one_evidence.v1' || !evidence.phases) { + throw new Error('The product evidence schema is unsupported.'); + } + const target = path.join(directory, name); + fs.writeFileSync(target, content, { mode: 0o600 }); + await artifact.uploadArtifact(`product-${name.slice(0, -5)}`, [target], directory, { retentionDays: 90 }); +} + +module.exports = { register, retain }; + +if (require.main === module) { + const core = require('@actions/core'); + Promise.resolve().then(() => { + if (core.getState('evidence-directory')) { + const { DefaultArtifactClient } = require('@actions/artifact'); + return retain(core, new DefaultArtifactClient()); + } + register(core); + }).catch(error => core.setFailed(`Product evidence was not retained: ${error.message}`)); +} diff --git a/.github/actions/retain-product-evidence/package-lock.json b/.github/actions/retain-product-evidence/package-lock.json new file mode 100644 index 000000000..a1861679e --- /dev/null +++ b/.github/actions/retain-product-evidence/package-lock.json @@ -0,0 +1,2527 @@ +{ + "name": "benchmark-product-evidence", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "benchmark-product-evidence", + "dependencies": { + "@actions/artifact": "5.0.0", + "@actions/core": "2.0.0" + } + }, + "node_modules/@actions/artifact": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@actions/artifact/-/artifact-5.0.0.tgz", + "integrity": "sha512-PboymiO3U0QEi50gVPO7omhwlJ24yYwVCq9hyWI5x7qab6PqF5mFErdp5cIGigHYswZm+vACH4i7Xc6WV8RA9w==", + "license": "MIT", + "dependencies": { + "@actions/core": "^2.0.0", + "@actions/github": "^6.0.1", + "@actions/http-client": "^3.0.0", + "@azure/core-http": "^3.0.5", + "@azure/storage-blob": "^12.15.0", + "@octokit/core": "^5.2.1", + "@octokit/plugin-request-log": "^1.0.4", + "@octokit/plugin-retry": "^3.0.9", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "archiver": "^7.0.1", + "jwt-decode": "^3.1.2", + "unzip-stream": "^0.3.1" + } + }, + "node_modules/@actions/core": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-2.0.0.tgz", + "integrity": "sha512-iGW52/zqhPUFnYl0s1ioXfJu86LGs7b+GYuO38JMPpsh14FQrNj3n2JBpC+vZ2CFS4lERQyn5koLDopY+6V/PQ==", + "license": "MIT", + "dependencies": { + "@actions/exec": "^1.1.1", + "@actions/http-client": "^3.0.0" + } + }, + "node_modules/@actions/exec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", + "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", + "license": "MIT", + "dependencies": { + "@actions/io": "^1.0.1" + } + }, + "node_modules/@actions/github": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.1.tgz", + "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==", + "license": "MIT", + "dependencies": { + "@actions/http-client": "^2.2.0", + "@octokit/core": "^5.0.1", + "@octokit/plugin-paginate-rest": "^9.2.2", + "@octokit/plugin-rest-endpoint-methods": "^10.4.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "undici": "^5.28.5" + } + }, + "node_modules/@actions/github/node_modules/@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "license": "MIT", + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + } + }, + "node_modules/@actions/http-client": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-3.0.2.tgz", + "integrity": "sha512-JP38FYYpyqvUsz+Igqlc/JG6YO9PaKuvqjM3iGvaLqFnJ7TFmcLyy2IDrY0bI0qCQug8E9K+elv5ZNfw62ZJzA==", + "license": "MIT", + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^6.23.0" + } + }, + "node_modules/@actions/http-client/node_modules/undici": { + "version": "6.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", + "integrity": "sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/@actions/io": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", + "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==", + "license": "MIT" + }, + "node_modules/@azure/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client/node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-http": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.5.tgz", + "integrity": "sha512-T8r2q/c3DxNu6mEJfPuJtptUVqwchxzjj32gKcnMi06rdiVONS9rar7kT9T2Am+XvER7uOzpsP79WsqNbdgdWg==", + "deprecated": "This package is no longer supported. Please refer to https://github.com/Azure/azure-sdk-for-js/blob/490ce4dfc5b98ba290dee3b33a6d0876c5f138e2/sdk/core/README.md", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", + "@azure/logger": "^1.0.0", + "@types/node-fetch": "^2.5.0", + "@types/tunnel": "^0.0.3", + "form-data": "^4.0.0", + "node-fetch": "^2.6.7", + "process": "^0.11.10", + "tslib": "^2.2.0", + "tunnel": "^0.0.6", + "uuid": "^8.3.0", + "xml2js": "^0.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.5.0.tgz", + "integrity": "sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/core-http-compat/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-lro/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.7.0.tgz", + "integrity": "sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.0.0-preview.13", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz", + "integrity": "sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.0.1", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.6.0.tgz", + "integrity": "sha512-e7lX/dk//F6Qf7BB6PTY4+p2yuOQtyOeHGyapYHNwqSp2OnYpwQt49A/Nin2XmKBQ69pwagR4k/lQBq8lbHQkA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.5.9", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.34.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.34.0.tgz", + "integrity": "sha512-cXDTCAMwDx8Ng9yUe82kOOd2kPnZ8AQqe4FkLMcpaQ3Mvr3/PXVdvRhyP0FtY5/U5Oh6JGk3EcEibaFOymRLFw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.5.0", + "apache-arrow": "^21.1.0", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob/node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.5.0.tgz", + "integrity": "sha512-bttzuhQiCIwrkzjPDA+AtAR7dg19L/CC6ztcqJ5LfvWpXuys9mHp0UQ0udYnoUvv9SCT9KTR5kqFvFr0e6k0lQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.4.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz", + "integrity": "sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@bufbuild/protoplugin": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protoplugin/-/protoplugin-2.16.0.tgz", + "integrity": "sha512-3FdYJys34VhFnBQsjiV9wurBkX/DBACQAkAk+nCKdO1p8ROVNi7gmaRAcba+4qq8hQq8rYVukuutwvGCvd4kJA==", + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "2.16.0", + "@typescript/vfs": "^1.6.2", + "typescript": "5.4.5" + } + }, + "node_modules/@bufbuild/protoplugin/node_modules/typescript": { + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", + "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@nodable/entities": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.1.0.tgz", + "integrity": "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.2.tgz", + "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", + "license": "MIT", + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/endpoint": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", + "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", + "license": "MIT", + "dependencies": { + "@octokit/request": "^8.4.1", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@octokit/plugin-request-log": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", + "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==", + "license": "MIT", + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@octokit/plugin-retry": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz", + "integrity": "sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^6.0.3", + "bottleneck": "^2.15.3" + } + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": { + "version": "12.11.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/types": { + "version": "6.41.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", + "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^12.11.0" + } + }, + "node_modules/@octokit/request": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^24.2.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobuf-ts/plugin": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/@protobuf-ts/plugin/-/plugin-2.11.1.tgz", + "integrity": "sha512-HyuprDcw0bEEJqkOWe1rnXUP0gwYLij8YhPuZyZk6cJbIgc/Q0IFgoHQxOXNIXAcXM4Sbehh6kjVnCzasElw1A==", + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "^2.4.0", + "@bufbuild/protoplugin": "^2.4.0", + "@protobuf-ts/protoc": "^2.11.1", + "@protobuf-ts/runtime": "^2.11.1", + "@protobuf-ts/runtime-rpc": "^2.11.1", + "typescript": "^3.9" + }, + "bin": { + "protoc-gen-dump": "bin/protoc-gen-dump", + "protoc-gen-ts": "bin/protoc-gen-ts" + } + }, + "node_modules/@protobuf-ts/protoc": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/@protobuf-ts/protoc/-/protoc-2.11.1.tgz", + "integrity": "sha512-mUZJaV0daGO6HUX90o/atzQ6A7bbN2RSuHtdwo8SSF2Qoe3zHwa4IHyCN1evftTeHfLmdz+45qo47sL+5P8nyg==", + "license": "Apache-2.0", + "bin": { + "protoc": "protoc.js" + } + }, + "node_modules/@protobuf-ts/runtime": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.11.1.tgz", + "integrity": "sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@protobuf-ts/runtime-rpc": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/@protobuf-ts/runtime-rpc/-/runtime-rpc-2.11.1.tgz", + "integrity": "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ==", + "license": "Apache-2.0", + "dependencies": { + "@protobuf-ts/runtime": "^2.11.1" + } + }, + "node_modules/@types/node": { + "version": "26.6.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.4.tgz", + "integrity": "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, + "node_modules/@types/tunnel": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", + "integrity": "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@typescript/vfs": { + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/@typescript/vfs/-/vfs-1.6.5.tgz", + "integrity": "sha512-gobLHQLmvQGJInFk31NvgMd1uYzRXsd8qgnOssyLD055R2QPnw1ubRbBQ4+T+5qC7IQVuYXdvNDE2G7tMDVHgw==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3" + }, + "peerDependencies": { + "typescript": "*" + } + }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/apache-arrow": { + "version": "21.2.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.2.0.tgz", + "integrity": "sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^25.2.0", + "flatbuffers": "^25.1.24", + "json-with-bigint": "^3.5.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "25.9.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.9.tgz", + "integrity": "sha512-b4e2xxj/yMeT2hNlD6x7zNFiQK2Dlmkb9CQ6v3/1G2RezKN4YS2lcEv8lzGm1mhs4f2lM4yNAMI/zUDBZBBt7w==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT" + }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/b4a": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.9.0.tgz", + "integrity": "sha512-dpfcF9fDNR6++cthXR67iyhgqWy9CBouAvIWhIntzBG6cvK/cnIPiZQjBwi/ZqjjBEDGfoNDtmB0kTjroOJ3pQ==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz", + "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.8.2.tgz", + "integrity": "sha512-+ZI68KHMUvosXfKbg/UOHK0tbCdRnegbvPEdEcZ3Nd6TetieQsJPRXBRXPdLyy8+3VSEbPXtsumTpEtt78xv9w==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.28.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-path": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.1.2.tgz", + "integrity": "sha512-ZyKbsuuqK6Ag0K8pX6V5Txq6XeJRvY+wXucnFGRjiyVYP9YWDpIQugk/b+enRYrEYBJaqLzghRQpXPMR7341Nw==", + "license": "Apache-2.0" + }, + "node_modules/bare-stream": { + "version": "2.13.4", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.4.tgz", + "integrity": "sha512-PcrQ8lVLbiJscNm1Kez+Yp4Gy4AHGcN1lzwjvf5NybWen7VvEgUfyfnXYJ2zNqWnzOfCb1Abq6lH8ti0syQszA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.5.4.tgz", + "integrity": "sha512-Gxa7UVWBr0/edU1b+TJhn/AZvMQUj9OGspvYsaTYQrAbZA4BOTZGL3LiZxvD+CeMlDH4juwD84+eTAp/bLYW5g==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", + "license": "Apache-2.0" + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "license": "ISC" + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.2", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.2.tgz", + "integrity": "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.1", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/json-with-bigint": { + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.12.tgz", + "integrity": "sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==", + "license": "MIT" + }, + "node_modules/jwt-decode": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.1.2.tgz", + "integrity": "sha512-UfpWE/VZn0iP50d8cz9NrZLM9lSWhcJ+0Gt/nm4by88UL+J1SiKN8/5dkjMmbEzwL2CAe+67GsegCbIKtbp75A==", + "license": "MIT" + }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/streamx": { + "version": "2.28.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.1.tgz", + "integrity": "sha512-zEzXb0s5Cds7tqMH6rhZ05lcJydCWiQPEwiNngVqzsxCc962vLY4Uw+mW7od8kDH258k2Uz/JrOkdIAAhSh9VA==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/tar-stream": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.2.tgz", + "integrity": "sha512-+8NeqHRjQWH9nYlwo2gamAMImZCVzI4UoEgDpWorBt9OEfppiZn+uSkskzQKPWIZyji/C8fpWO7u69G0DX0tbg==", + "license": "MIT", + "dependencies": { + "b4a": "^1.9.0", + "bare-fs": "^4.8.2", + "fast-fifo": "^1.3.2", + "streamx": "^2.28.1" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "license": "MIT", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, + "node_modules/typescript": { + "version": "3.9.10", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", + "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, + "node_modules/universal-user-agent": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", + "license": "ISC" + }, + "node_modules/unzip-stream": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.4.tgz", + "integrity": "sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==", + "license": "MIT", + "dependencies": { + "binary": "^0.3.0", + "mkdirp": "^0.5.1" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + } + } +} diff --git a/.github/actions/retain-product-evidence/package.json b/.github/actions/retain-product-evidence/package.json new file mode 100644 index 000000000..c324c2456 --- /dev/null +++ b/.github/actions/retain-product-evidence/package.json @@ -0,0 +1,8 @@ +{ + "name": "benchmark-product-evidence", + "private": true, + "dependencies": { + "@actions/artifact": "5.0.0", + "@actions/core": "2.0.0" + } +} diff --git a/.github/workflows/guardrails.yml b/.github/workflows/guardrails.yml index 8bb761e76..3d69c3ecb 100644 --- a/.github/workflows/guardrails.yml +++ b/.github/workflows/guardrails.yml @@ -23,6 +23,8 @@ jobs: - name: Run Ruby tests run: bundle exec ruby -Itest -e 'Dir["test/*_test.rb"].sort.each { |file| require_relative file }' + - name: Check product evidence retention + run: node --test test/product_evidence_test.cjs - name: Lint workflows run: |- go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 diff --git a/.gitignore b/.gitignore index 9f2e84857..d9a659e8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store __pycache__/ +node_modules/ diff --git a/bin/bench b/bin/bench index fadab9940..3d68c5c72 100755 --- a/bin/bench +++ b/bin/bench @@ -14,7 +14,7 @@ id = ARGV.first unless ARGV.first&.start_with?("--") ARGV.shift if id options = {lane: "fresh", inputs: {}, ref: "main"} parser = OptionParser.new do |value| - value.banner = "Usage: bin/bench [case-id] [options]" + value.banner = "Usage: bin/bench [case-id] [options]" value.on("--lane LANE") { |input| options[:lane] = input } value.on("--workflow NAME") { |input| options[:workflow] = input } value.on("--input NAME=VALUE") do |input| @@ -97,11 +97,16 @@ begin raise BenchmarkCases::Error, "Select a case ID and --directory" unless item && options[:directory] BenchmarkCases.update_source(item, options.fetch(:directory)) puts "Updated #{item.fetch('id')} source pins" - when "finish" + when "finish", "collect" raise BenchmarkCases::Error, "Select a case ID, --series, --sample, and --directory for preserved evidence" unless item && options.values_at(:series, :sample, :directory).all? directory = File.join(BenchmarkCases::ROOT, "results", id, options.fetch(:series)) raise BenchmarkCases::Error, "Series belongs to another case" unless BenchmarkSeries.load(directory).fetch("case_id") == id - puts JSON.pretty_generate(BenchmarkSeries.finish(directory, evidence_directory: options.fetch(:directory), sample: options.fetch(:sample))) + if command == "collect" + require_relative "../scripts/collect-series" + puts JSON.pretty_generate(CollectSeries.call(directory, evidence_directory: options.fetch(:directory), sample: options.fetch(:sample))) + else + puts JSON.pretty_generate(BenchmarkSeries.finish(directory, evidence_directory: options.fetch(:directory), sample: options.fetch(:sample))) + end when "report" raise BenchmarkCases::Error, "Select a case ID and --series" unless item && options[:series] directory = options[:directory] || File.join(BenchmarkCases::ROOT, "results", id, options.fetch(:series)) diff --git a/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml b/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml index f5117c27e..86e46e461 100644 --- a/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml +++ b/cases/chroma/payload/.github/actions/chroma-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Chroma Docker benchmark phase description: Runs one Chroma Docker cache provider and phase. inputs: @@ -49,24 +50,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}" - else - cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -80,7 +64,8 @@ runs: PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/chroma-benchmark:boringcache - run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" + run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" + --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -88,29 +73,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: docker/chroma.Dockerfile @@ -120,47 +85,24 @@ runs: push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" load: "${{ inputs.load_image == 'true' }}" build-args: RELEASE_MODE=1 - provenance: false - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - working-directory: "." - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + provenance: 'false' + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - working-directory: "." - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Docker output id: output_verification shell: bash env: STRATEGY: "${{ inputs.strategy }}" LOAD_IMAGE: "${{ inputs.load_image }}" - PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" IMAGE: ghcr.io/${{ github.repository_owner }}/chroma-benchmark:actions-cache - run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -171,13 +113,11 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" SCCACHE_PROOF: "${{ inputs.strategy == 'boringcache' && 'true' || '' }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml b/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml index d1d826b86..fa76b0313 100644 --- a/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml +++ b/cases/duckgres/payload/.github/actions/duckgres-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Duckgres Docker benchmark phase description: Runs one Duckgres Docker cache provider and phase. inputs: @@ -49,24 +50,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}" - else - cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -90,29 +74,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: upstream/Dockerfile @@ -124,45 +88,24 @@ runs: VERSION=build-${{ steps.scope.outputs.source_sha }} COMMIT=${{ steps.scope.outputs.source_sha }} BUILD_TAGS=kubernetes - provenance: false - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + provenance: 'false' + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Docker output id: output_verification shell: bash env: STRATEGY: "${{ inputs.strategy }}" LOAD_IMAGE: "${{ inputs.load_image }}" - PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" IMAGE: ghcr.io/${{ github.repository_owner }}/duckgres-benchmark:actions-cache - run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -173,12 +116,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/hugo-go/payload/.github/actions/hugo-go-benchmark/action.yml b/cases/hugo-go/payload/.github/actions/hugo-go-benchmark/action.yml index 42721c606..cf892a3bd 100644 --- a/cases/hugo-go/payload/.github/actions/hugo-go-benchmark/action.yml +++ b/cases/hugo-go/payload/.github/actions/hugo-go-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Hugo Go benchmark phase description: Runs one Hugo Go cache provider and phase. inputs: @@ -39,26 +40,10 @@ runs: shell: bash env: CACHE_LANE: "${{ inputs.cache_lane }}" + BENCHMARK_ID: "${{ inputs.benchmark_id }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="hugo-go-rolling-${ref_slug}" - else - cache_scope="hugo-go-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" source_sha="$(git rev-parse HEAD:upstream)" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" @@ -95,7 +80,7 @@ runs: - name: Publish with BoringCache id: boringcache-publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: go @@ -106,7 +91,7 @@ runs: - name: Restore with BoringCache id: boringcache-restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: go diff --git a/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml b/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml index 03d1e0b97..2219650aa 100644 --- a/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml +++ b/cases/hugo/payload/.github/actions/hugo-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Hugo Docker benchmark phase description: Runs one Hugo Docker cache provider and phase. inputs: @@ -52,20 +53,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}"; else cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}"; fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -79,7 +67,8 @@ runs: PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" IMAGE: ghcr.io/${{ github.repository_owner }}/hugo-benchmark:boringcache - run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" --load "$LOAD_IMAGE" + run: ruby ./scripts/activate-docker-plan.rb --push "$PUSH_IMAGE" --image "$IMAGE" + --load "$LOAD_IMAGE" - name: Log in to the GitHub Container Registry if: inputs.push_image == 'true' && github.ref == 'refs/heads/main' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 @@ -87,29 +76,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: upstream/Dockerfile @@ -119,46 +88,24 @@ runs: build-args: HUGO_BUILD_TAGS=extended,withdeploy provenance: "${{ inputs.load_image != 'true' && 'mode=max' || 'false' }}" sbom: "${{ inputs.load_image != 'true' }}" - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - cli-platform: "${{ inputs.cli_platform }}" - mode: docker - trust-policy: publish - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" + buildkit-image: "${{ inputs.buildkit_image }}" cli-platform: "${{ inputs.cli_platform }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" - name: Verify the Docker output id: output_verification shell: bash env: STRATEGY: "${{ inputs.strategy }}" LOAD_IMAGE: "${{ inputs.load_image }}" - PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" IMAGE: ghcr.io/${{ github.repository_owner }}/hugo-benchmark:actions-cache - run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -169,12 +116,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/immich/payload/.github/actions/immich-base-images-benchmark/action.yml b/cases/immich/payload/.github/actions/immich-base-images-benchmark/action.yml index 4672580a9..d23e7c4db 100644 --- a/cases/immich/payload/.github/actions/immich-base-images-benchmark/action.yml +++ b/cases/immich/payload/.github/actions/immich-base-images-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Immich base-images benchmark phase description: Builds Immich's pinned native dependency graph with one cache provider. inputs: @@ -72,59 +73,19 @@ runs: TOOL_CACHE: "${{ inputs.strategy == 'boringcache-ccache' }}" run: ruby ./scripts/activate-docker-plan.rb --scenario base-images --tool-cache "$TOOL_CACHE" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: base-images-upstream/server file: base-images-upstream/server/Dockerfile platforms: linux/amd64 target: libvips - push: false - cache-from: "${{ inputs.phase == 'rebuild' && format('type=gha,scope={0}', steps.scope.outputs.cache_scope) - || '' }}" - cache-to: "${{ inputs.phase == 'seed' && format('type=gha,scope={0},mode=max', - steps.scope.outputs.cache_scope) || '' }}" - - name: Publish the BoringCache seed - id: publish - if: startsWith(inputs.strategy, 'boringcache') && inputs.phase == 'seed' - uses: ./.github/actions/boringcache - with: - mode: docker - trust-policy: publish - - name: Restore the BoringCache rebuild - id: restore - if: startsWith(inputs.strategy, 'boringcache') && inputs.phase == 'rebuild' - uses: ./.github/actions/boringcache - with: - mode: docker - trust-policy: restore - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + push: 'false' + strategy: "${{ startsWith(inputs.strategy, 'boringcache') && 'boringcache' || + inputs.strategy }}" + phase: "${{ inputs.phase == 'seed' && 'publish' || 'warm' }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" - name: Write the benchmark phase evidence shell: bash env: @@ -132,12 +93,10 @@ runs: STRATEGY: "${{ inputs.strategy }}" REPORT_PHASE: "${{ inputs.phase == 'seed' && 'cold' || 'warm' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" diff --git a/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml b/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml index 3ab1cd929..fe4ec96e2 100644 --- a/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml +++ b/cases/immich/payload/.github/actions/immich-docker-benchmark/action.yml @@ -54,20 +54,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}"; else cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}"; fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -94,29 +81,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: "./.github/actions/setup-docker" - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: upstream/server/Dockerfile @@ -129,36 +96,13 @@ runs: BUILD_SOURCE_REF=${{ github.ref_name }} BUILD_SOURCE_COMMIT=${{ steps.scope.outputs.source_sha }} DEVICE=cpu - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max load: "${{ inputs.load_image == 'true' }}" - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Docker output id: output_verification shell: bash @@ -179,12 +123,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml b/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml index 886a1dc16..234867758 100644 --- a/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml +++ b/cases/linkerd2/payload/.github/actions/linkerd2-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Linkerd2 Docker benchmark phase description: Runs one Linkerd2 Docker cache provider and phase. inputs: @@ -49,20 +50,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}"; else cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}"; fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -87,29 +75,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: upstream/web/Dockerfile @@ -117,45 +85,24 @@ runs: push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" load: "${{ inputs.load_image == 'true' }}" build-args: LINKERD_VERSION=${{ steps.scope.outputs.source_tag }} - provenance: false - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + provenance: 'false' + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Docker output id: output_verification shell: bash env: STRATEGY: "${{ inputs.strategy }}" LOAD_IMAGE: "${{ inputs.load_image }}" - PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" + PUSH_IMAGE: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' + }}" IMAGE: ghcr.io/${{ github.repository_owner }}/linkerd2-benchmark:actions-cache - run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" --push "$PUSH_IMAGE" --image "$IMAGE" + run: ruby ./scripts/verify-docker-output.rb --strategy "$STRATEGY" --load "$LOAD_IMAGE" + --push "$PUSH_IMAGE" --image "$IMAGE" - name: Write the benchmark phase evidence shell: bash env: @@ -166,12 +113,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml b/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml index c57cb168a..25fde6663 100644 --- a/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml +++ b/cases/mastodon/payload/.github/actions/mastodon-docker-benchmark/action.yml @@ -69,20 +69,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}"; else cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}"; fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -112,29 +99,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: "./.github/actions/setup-docker" - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: "${{ inputs.workload == 'server' && inputs.docker_tool_cache == 'true' @@ -148,37 +115,14 @@ runs: MASTODON_VERSION_PRERELEASE=${{ steps.scope.outputs.version_prerelease }} MASTODON_VERSION_METADATA= SOURCE_COMMIT=${{ steps.scope.outputs.source_sha }} - provenance: false - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max + provenance: 'false' load: "${{ inputs.load_image == 'true' }}" - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Docker output id: output_verification shell: bash @@ -200,12 +144,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml b/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml index bab5e0aae..3bdde6b20 100644 --- a/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml +++ b/cases/n8n/payload/.github/actions/n8n-docker-benchmark/action.yml @@ -72,24 +72,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}" - else - cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -141,29 +124,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: "./.github/actions/setup-docker" - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: "${{ inputs.dockerfile_path }}" @@ -174,40 +137,15 @@ runs: NODE_VERSION=${{ inputs.node_version }} N8N_VERSION=snapshot N8N_RELEASE_TYPE=dev - provenance: false - sbom: false - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max + provenance: 'false' + sbom: 'false' load: "${{ inputs.load_image == 'true' }}" - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - working-directory: "." - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - working-directory: "." - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Docker output id: output_verification shell: bash @@ -228,12 +166,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml b/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml index 2328ce636..710838425 100644 --- a/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml +++ b/cases/n8n/payload/.github/actions/n8n-turbo-benchmark/action.yml @@ -59,24 +59,7 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}" - else - cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" source_sha="$(git rev-parse HEAD:upstream)" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" diff --git a/cases/opentelemetry-java/payload/.github/actions/opentelemetry-gradle-benchmark/action.yml b/cases/opentelemetry-java/payload/.github/actions/opentelemetry-gradle-benchmark/action.yml index 3c928612d..a77799f19 100644 --- a/cases/opentelemetry-java/payload/.github/actions/opentelemetry-gradle-benchmark/action.yml +++ b/cases/opentelemetry-java/payload/.github/actions/opentelemetry-gradle-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run OpenTelemetry Gradle benchmark phase description: Runs one OpenTelemetry Java Gradle cache provider and phase. inputs: @@ -39,26 +40,10 @@ runs: shell: bash env: CACHE_LANE: "${{ inputs.cache_lane }}" + BENCHMARK_ID: "${{ inputs.benchmark_id }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="otel-gradle-rolling-${ref_slug}" - else - cache_scope="otel-gradle-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" source_sha="$(git rev-parse HEAD:upstream)" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" @@ -101,7 +86,7 @@ runs: - name: Publish with BoringCache id: boringcache-publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: gradle @@ -110,7 +95,7 @@ runs: - name: Restore with BoringCache id: boringcache-restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: gradle diff --git a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml index 931361e86..6a89293b2 100644 --- a/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml +++ b/cases/posthog/payload/.github/actions/posthog-docker-benchmark/action.yml @@ -132,28 +132,7 @@ runs: PLATFORM: "${{ inputs.platform }}" run: | set -euo pipefail - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - elif [[ "$PHASE" == "warm" ]]; then - [[ -n "$PUBLISHED_SCOPE" ]] - cache_scope="$PUBLISHED_SCOPE" - else - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}" - else - cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope --require-published)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" source_url="$(git config -f .gitmodules submodule.upstream.url)" @@ -192,29 +171,9 @@ runs: registry: ghcr.io username: "${{ github.actor }}" password: "${{ github.token }}" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the Actions cache protocol - if: inputs.strategy == 'actions-cache' || inputs.strategy == 'runs-on-cache' - uses: "./.github/actions/setup-docker" - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the Actions cache protocol - if: inputs.strategy == 'actions-cache' || inputs.strategy == 'runs-on-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: "${{ steps.dockerfile.outputs.path }}" @@ -223,43 +182,19 @@ runs: || 'actions-cache' }} push: "${{ inputs.push_image == 'true' && github.ref == 'refs/heads/main' }}" load: "${{ inputs.load_image == 'true' }}" - provenance: false + provenance: 'false' secrets: "${{ inputs.posthog_sourcemap_api_key != '' && format('posthog_upload_sourcemaps_cli_api_key={0}', inputs.posthog_sourcemap_api_key) || '' }}" - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && (inputs.phase == 'publish' || (inputs.phase - == 'warm' && inputs.publish_on_warm == 'true')) - uses: "./.github/actions/boringcache" - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - POSTHOG_SOURCEMAP_API_KEY: "${{ inputs.posthog_sourcemap_api_key }}" - with: + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - fail-on-cache-miss: "${{ inputs.phase == 'warm' }}" - - name: Build from the BoringCache cache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' && inputs.publish_on_warm - != 'true' - uses: "./.github/actions/boringcache" + buildkit-image: "${{ inputs.buildkit_image }}" + publish-on-warm: "${{ inputs.publish_on_warm }}" + no-cache: "${{ inputs.no_cache }}" env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" POSTHOG_SOURCEMAP_API_KEY: "${{ inputs.posthog_sourcemap_api_key }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" - name: Verify the Docker output id: output_verification shell: bash @@ -283,12 +218,10 @@ runs: 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" REPORT_VARIANT: "${{ inputs.report_variant || steps.scope.outputs.variant }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_REPOSITORY: "${{ steps.scope.outputs.source_repository }}" diff --git a/cases/qdrant/payload/.github/actions/qdrant-docker-benchmark/action.yml b/cases/qdrant/payload/.github/actions/qdrant-docker-benchmark/action.yml index ea8838fde..e286b73c4 100644 --- a/cases/qdrant/payload/.github/actions/qdrant-docker-benchmark/action.yml +++ b/cases/qdrant/payload/.github/actions/qdrant-docker-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Qdrant Docker benchmark phase description: Runs one Qdrant Docker cache provider and phase. inputs: @@ -46,85 +47,29 @@ runs: CACHE_LANE: "${{ inputs.cache_lane }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then cache_scope="${BENCHMARK_ID}-rolling-${ref_slug}"; else cache_scope="${BENCHMARK_ID}-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}"; fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" echo "source_sha=$(git -C upstream rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Start the provider setup timer - id: setup_timer - shell: bash - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Set up Buildx for the GitHub Actions comparison - if: inputs.strategy == 'actions-cache' - uses: ./.github/actions/setup-docker - - name: Start the build timer - id: build_timer - shell: bash - env: - SETUP_STARTED_AT: "${{ steps.setup_timer.outputs.started_at }}" - run: | - set -euo pipefail - now="$(date +%s)" - echo "setup_seconds=$(( now - SETUP_STARTED_AT ))" >> "$GITHUB_OUTPUT" - echo "started_at=${now}" >> "$GITHUB_OUTPUT" - - name: Build with the GitHub Actions cache - if: inputs.strategy == 'actions-cache' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 - env: - DOCKER_BUILD_SUMMARY: 'false' - DOCKER_BUILD_RECORD_UPLOAD: 'false' + - name: Build with the declared cache provider + id: provider_build + uses: "./.github/actions/docker-benchmark" with: context: upstream file: upstream/Dockerfile tags: qdrant/qdrant:e2e-tests - push: false + push: 'false' build-args: | PROFILE=ci FEATURES=data-consistency-check,staging - load: true - provenance: false - cache-from: type=gha,scope=${{ steps.scope.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.scope.outputs.cache_scope }},mode=max - - name: Publish with BoringCache - id: publish - if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: - cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: publish - - name: Restore with BoringCache - id: restore - if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache - env: - BORINGCACHE_MANAGED_BUILDKIT_IMAGE: "${{ inputs.buildkit_image }}" - with: + load: 'true' + provenance: 'false' + strategy: "${{ inputs.strategy == 'runs-on-cache' && 'actions-cache' || inputs.strategy + }}" + phase: "${{ inputs.phase }}" + cache-scope: "${{ steps.scope.outputs.cache_scope }}" cli-version: "${{ inputs.cli_version }}" - mode: docker - trust-policy: restore - fail-on-cache-miss: true - - name: Capture the build timing - id: build_timing - shell: bash - env: - BUILD_STARTED_AT: "${{ steps.build_timer.outputs.started_at }}" - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" + buildkit-image: "${{ inputs.buildkit_image }}" - name: Verify the Qdrant image shell: bash run: docker image inspect qdrant/qdrant:e2e-tests >/dev/null @@ -137,12 +82,10 @@ runs: REPORT_PHASE: "${{ inputs.cache_lane == 'rolling' && 'commit' || inputs.phase == 'warm' && 'warm' || 'cold' }}" WORKFLOW_STARTED_AT: "${{ steps.workflow_timer.outputs.started_at }}" - SETUP_SECONDS: "${{ steps.build_timer.outputs.setup_seconds }}" - BUILD_SECONDS: "${{ steps.build_timing.outputs.build_seconds }}" - CACHE_HIT: "${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit - }}" - EVIDENCE_PATH: "${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path - }}" + SETUP_SECONDS: "${{ steps.provider_build.outputs.setup-seconds }}" + BUILD_SECONDS: "${{ steps.provider_build.outputs.build-seconds }}" + CACHE_HIT: "${{ steps.provider_build.outputs.cache-hit }}" + EVIDENCE_PATH: "${{ steps.provider_build.outputs.evidence-path }}" STORAGE_KEY: "${{ steps.scope.outputs.cache_scope }}" GITHUB_TOKEN: "${{ github.token }}" SOURCE_SHA: "${{ steps.scope.outputs.source_sha }}" diff --git a/cases/spring-ai/payload/.github/actions/spring-maven-benchmark/action.yml b/cases/spring-ai/payload/.github/actions/spring-maven-benchmark/action.yml index c443b1562..6b3227647 100644 --- a/cases/spring-ai/payload/.github/actions/spring-maven-benchmark/action.yml +++ b/cases/spring-ai/payload/.github/actions/spring-maven-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Spring AI Maven benchmark phase description: Runs one Spring AI Maven cache provider and phase. inputs: @@ -39,26 +40,10 @@ runs: shell: bash env: CACHE_LANE: "${{ inputs.cache_lane }}" + BENCHMARK_ID: "${{ inputs.benchmark_id }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="spring-ai-maven-rolling-${ref_slug}" - else - cache_scope="spring-ai-maven-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" source_sha="$(git rev-parse HEAD:upstream)" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" @@ -101,7 +86,7 @@ runs: - name: Publish with BoringCache id: boringcache-publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: maven @@ -110,7 +95,7 @@ runs: - name: Restore with BoringCache id: boringcache-restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: maven diff --git a/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml b/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml index ddd1e112a..5b5d08bf0 100644 --- a/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml +++ b/cases/storybook/payload/.github/actions/storybook-nx-benchmark/action.yml @@ -1,3 +1,4 @@ +--- name: Run Storybook Nx benchmark phase description: Runs one Storybook Nx cache provider and phase. inputs: @@ -39,26 +40,10 @@ runs: shell: bash env: CACHE_LANE: "${{ inputs.cache_lane }}" + BENCHMARK_ID: "${{ inputs.benchmark_id }}" run: | set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="storybook-rolling-${ref_slug}" - else - cache_scope="storybook-run-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - if [[ -n "${BENCHMARK_SERIES_ID:-}" ]]; then - [[ "$BENCHMARK_SERIES_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || exit 1 - [[ "${BENCHMARK_SAMPLE:-}" =~ ^[1-9][0-9]*$ ]] || exit 1 - if [[ "$CACHE_LANE" == "rolling" ]]; then - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}" - else - cache_scope="${BENCHMARK_ID:-benchmark}-series-${BENCHMARK_SERIES_ID}-s${BENCHMARK_SAMPLE}-r${GITHUB_RUN_ID}-a${GITHUB_RUN_ATTEMPT}" - fi - fi + cache_scope="$(ruby ./scripts/benchmark-phase.rb scope)" ruby ./scripts/scope-case-cache.rb "$cache_scope" source_sha="$(git rev-parse HEAD:upstream)" echo "cache_scope=${cache_scope}" >> "$GITHUB_OUTPUT" @@ -66,7 +51,7 @@ runs: echo "source_repository=storybookjs/storybook" >> "$GITHUB_OUTPUT" echo "source_sha=${source_sha}" >> "$GITHUB_OUTPUT" - name: Set up Node.js - uses: ./.github/actions/setup-node + uses: "./.github/actions/setup-node" with: node-version: "${{ inputs.node_version }}" - name: Enable the pinned Yarn release @@ -104,7 +89,7 @@ runs: - name: Publish with BoringCache id: boringcache-publish if: inputs.strategy == 'boringcache' && inputs.phase == 'publish' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: archive @@ -114,7 +99,7 @@ runs: - name: Restore with BoringCache id: boringcache-restore if: inputs.strategy == 'boringcache' && inputs.phase == 'warm' - uses: ./.github/actions/boringcache + uses: "./.github/actions/boringcache" with: cli-version: "${{ inputs.cli_version }}" mode: archive diff --git a/data/latest/series.json b/data/latest/series.json index c4860b050..0d771ee7e 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -36,9 +36,12 @@ "definition_sha256": "6a50efe5eda093d17fd5abc3d6967183cd287e3b2203b8876d83b1937d3b7301", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -1820,9 +1823,12 @@ "definition_sha256": "1b406d174de4de92e0c62265436e2e130d2c04966378bab14e734ff70ec50044", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -2727,34 +2733,73 @@ }, "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485", "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 270, + "min": 270, + "max": 270 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 223, + "min": 223, + "max": 223 + }, + "storage": { + "median": 411514484, + "min": 411514484, + "max": 411514484 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": { + "median": 411514484, + "min": 411514484, + "max": 411514484 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "hugo", @@ -2774,14 +2819,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37148378881" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148378881", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6560d3ffaa57bd746e57c74503987869c470c24ad4b4a02632b6392ad18239aa" + } + ], + "report_state": "current", + "report_path": "results/hugo/image-output-qualification-02/report.json", + "interpretation_path": "results/hugo/image-output-qualification-02/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37148378881, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148378881.tar", + "bundle_bytes": 263168, + "sha256": "de853e9d1c1b0b6bead7e1822a5efbc3a0bb92b15e37e1e53c400757b2354c56", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "hugo", @@ -2931,9 +3002,12 @@ "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -3080,9 +3154,12 @@ "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -3344,9 +3421,12 @@ "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -3492,34 +3572,73 @@ }, "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 52, + "min": 52, + "max": 52 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 21, + "min": 21, + "max": 21 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 40, + "min": 40, + "max": 40 + }, + "storage": { + "median": 106281953, + "min": 106281953, + "max": 106281953 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 22, + "min": 22, + "max": 22 + }, + "storage": { + "median": 106281953, + "min": 106281953, + "max": 106281953 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "mastodon", @@ -3540,14 +3659,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37149053291" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", + "sample": 1, + "run_id": "37149053291", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "530bf111ae6b4a6f70347e2e293b0ad75cec21dd175728be464cf8ecfc1ebfec" + } + ], + "report_state": "current", + "report_path": "results/mastodon/streaming-output-qualification-01/report.json", + "interpretation_path": "results/mastodon/streaming-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37149053291, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149053291.tar", + "bundle_bytes": 266240, + "sha256": "899c7adc2f77d143cfdda44d59b9f12884877c5a5fcf65bed1c5989341c8b2a9", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "n8n", @@ -3580,44 +3725,83 @@ }, "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." ], - "summaries": [], - "dispatches": [ + "missing_observations": [], + "summaries": [ { - "case_id": "n8n", - "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", - "lane": "fresh", - "inputs": { - "case_id": "n8n", - "cli_version": "v1.33.0", - "variant": "distroless", + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 214, + "min": 214, + "max": 214 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 132, + "min": 132, + "max": 132 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 81, + "min": 81, + "max": 81 + }, + "storage": { + "median": 581055738, + "min": 581055738, + "max": 581055738 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 110, + "min": 110, + "max": 110 + }, + "storage": { + "median": 581055738, + "min": 581055738, + "max": 581055738 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "n8n", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n", + "cli_version": "v1.33.0", + "variant": "distroless", "series_id": "distroless-output-qualification-01", "sample": "1" }, @@ -3628,14 +3812,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37149056843" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", + "sample": 1, + "run_id": "37149056843", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4bbc22e12b0a3e5e9c1c5c481f68f29002291e60a1bcc748ea3e18eaaed8566e" + } + ], + "report_state": "current", + "report_path": "results/n8n/distroless-output-qualification-01/report.json", + "interpretation_path": "results/n8n/distroless-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37149056843, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149056843.tar", + "bundle_bytes": 1658880, + "sha256": "b1b97061460c375759d37184868d53cddad9c1b980b4e69d8e6a67cce374ce09", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "n8n", @@ -3670,9 +3880,12 @@ "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -3818,34 +4031,73 @@ }, "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 144, + "min": 144, + "max": 144 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 89, + "min": 89, + "max": 89 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 62, + "min": 62, + "max": 62 + }, + "storage": { + "median": 413827357, + "min": 413827357, + "max": 413827357 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 60, + "min": 60, + "max": 60 + }, + "storage": { + "median": 413827357, + "min": 413827357, + "max": 413827357 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "n8n", @@ -3866,14 +4118,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37149055130" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "runners-output-qualification-01", + "sample": 1, + "run_id": "37149055130", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c52e6d5bedd2e1d5e4333c820e0895b30eee8eb5c5ec02389cb5d669dc6f7017" + } + ], + "report_state": "current", + "report_path": "results/n8n/runners-output-qualification-01/report.json", + "interpretation_path": "results/n8n/runners-output-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37149055130, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149055130.tar", + "bundle_bytes": 1602560, + "sha256": "36d2b8fe86fbfe42fa9df155468b0beda3b6a1baabec398dd3a577e04feafafd", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "n8n", @@ -4774,34 +5052,73 @@ }, "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad", "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1889, + "min": 1889, + "max": 1889 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 187, + "min": 187, + "max": 187 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 1016, + "min": 1016, + "max": 1016 + }, + "storage": { + "median": 6237909312, + "min": 6237909312, + "max": 6237909312 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 162, + "min": 162, + "max": 162 + }, + "storage": { + "median": 6237909312, + "min": 6237909312, + "max": 6237909312 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "posthog", @@ -4822,14 +5139,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37148377350" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", + "sample": 1, + "run_id": "37148377350", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7d2cf7f6e9b12c476c9b3e23b73d31fbe8f25176a91bf7d18fc29a027e819895" + } + ], + "report_state": "current", + "report_path": "results/posthog/image-output-combined-qualification-01/report.json", + "interpretation_path": "results/posthog/image-output-combined-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37148377350, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148377350.tar", + "bundle_bytes": 373248, + "sha256": "bf89e6b23b5517ca298ff620dde1885fce19ec9a9a8d92fce5092a2dfd93e893", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "posthog", @@ -4868,7 +5211,9 @@ "execution_verified": false, "methodology_issues": [ "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", - "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series.", + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." ], "missing_observations": [ [ @@ -5002,34 +5347,73 @@ }, "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc", "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", - "state": "requested", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1609, + "min": 1609, + "max": 1609 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 175, + "min": 175, + "max": 175 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 778, + "min": 778, + "max": 778 + }, + "storage": { + "median": 6237262375, + "min": 6237262375, + "max": 6237262375 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 136, + "min": 136, + "max": 136 + }, + "storage": { + "median": 6237262375, + "min": 6237262375, + "max": 6237262375 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "posthog", @@ -5050,14 +5434,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37148375181" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148375181", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ad47456c459688cb932e9d232c57790b64b694616f587246c3bb3b3c07e77e25" + } + ], + "report_state": "current", + "report_path": "results/posthog/image-output-qualification-02/report.json", + "interpretation_path": "results/posthog/image-output-qualification-02/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37148375181, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148375181.tar", + "bundle_bytes": 372736, + "sha256": "49df16499aee7a038dfea048f8ec82e605cbfa22fa8822e0060bc7ce213e65b4", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "qdrant", @@ -5092,9 +5502,12 @@ "definition_sha256": "e05d45d9cb64f12d4e8b734ca13bc253a3a03c7f2833add246c8981abe668bf7", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { diff --git a/docs/migration.md b/docs/migration.md index 9b9375770..09d8be0bb 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -66,6 +66,29 @@ The native Docker actions also use the shared Ruby source preparer. Chroma's [recipe review](../cases/chroma/recipe-review.md) records the removal of its old diagnostic source-layer marker from the identical-source replay. The replacement timings must not be combined with that earlier projection. + +The October 4 review found a shared fairness error in those Docker comparisons: +Actions Cache exported warm state while BoringCache restored without publishing. +Plan-bound reviews now mark all affected recorded Docker series ineligible for a +comparative claim, retaining the original measurements and successful output +checks. The shared `docker-benchmark` action now owns setup, timing, and the same +publication decision for both providers. Cold publishes; ordinary warm replay +does not. Explicit warm publication applies to both arms. + +The [lifecycle screening archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04) +preserves the six completed PostHog layers/combined, Hugo, Mastodon streaming, and +n8n runners/distroless runs requested on October 3. Their four phase records per +run are imported through `bin/bench collect`; each published bundle was downloaded +again and verified against its checksum and 13-file inventory. These runs retain +the same publication-policy limitation. They also lack the original final One +evidence envelope. The new provider post hook retains that envelope after product +cleanup. Corrected execution requires new frozen series and live qualification. + +Canary monitoring now follows the active historical schedules through explicit +registry metadata. Its October 4 read-only collection verified all 16 repository +receipts and 17 child runs successfully. Central dispatch remains manual until +each schedule is qualified and cut over. This fixes monitoring without treating +an imported definition as an active central schedule. n8n Turbo's primary measurement now captures cache setup and the build separately from its dependency installation. The install still runs before the build. Earlier results retain their original timed scope and must not be mixed with this series. diff --git a/docs/process.md b/docs/process.md index 253ed45ae..5b7a27e92 100644 --- a/docs/process.md +++ b/docs/process.md @@ -93,6 +93,22 @@ Reusable callers declare `contents: read`, `actions: read`, `packages: read`, an the workflow, including calls whose execution condition is false. Docker tool caches remain product-plan settings rather than a second implementation of the product's proxy or cache lifecycle. +Docker case actions call `.github/actions/docker-benchmark` for provider setup, +timing, and execution. One phase policy controls cache publication in both arms: +cold publishes; an identical-source warm replay restores without publishing; +an explicitly declared `publish-on-warm` observation publishes in both arms. +Image loading stays inside this action's timer. Output inspection runs after it. +Use `benchmark-phase.rb scope` for the shared case/series identity, and keep +tool-specific tags in the product plan. + +The provider wrapper registers `.github/actions/retain-product-evidence` before +calling One. Its GitHub post hook runs after One's cleanup, then uploads the +original final evidence JSON using GitHub's artifact SDK. Missing or invalid +evidence fails retention. The small JavaScript hook is required for GitHub post +steps; it does not interpret product measurements or implement cache behavior. +Its dependencies install during preparation, outside the comparison timer. +The Ruby reporter retains its structured measurements; the original envelope +remains a separate artifact so post-step status and future product fields survive. The harness Ruby version is pinned once in `.tool-versions`. Ruby setup reads that file from the harness checkout, including when a shared action is called from an older benchmark repository. Guardrails verify preparation on Ubuntu @@ -242,10 +258,15 @@ do not infer publication permission from a successful build or a cache hit. ```sh bin/bench preserve --repository boringcache/benchmarks --run --directory /tmp/evidence- bin/bench verify-evidence --directory /tmp/evidence- -bin/bench finish --series screening-01 --sample 1 --directory /tmp/evidence- -bin/bench report --series screening-01 +bin/bench collect --series screening-01 --sample 1 --directory /tmp/evidence- ``` +`collect` checks the verified export against the retained dispatch, imports the +original phase artifacts, checks completion, and regenerates the report. It +rejects conflicting records before importing them and can resume an identical +import. It never derives missing phase measurements from logs. `record`, `finish`, +and `report` remain available separately for reviewed historical imports. + Publish the verified bundle to durable storage with a stable link. Experimental cache retention is not permanent evidence retention. The exporter includes all available attempts and records gaps; partial exports remain partial. Keep @@ -276,6 +297,12 @@ evidence selection to an immutable report link. A green workflow or a case's existing publication status does not approve a new series. Replacing the legacy feed requires a separately qualified adapter for canonical series reports. +Canary monitoring follows `canary_location` in `suites/published.json`. Keep it +`historical` while the original repository owns the schedule. Set it to `central` +only when its central dispatcher and receipt have been qualified and its old +schedule retired. Missing, stale, or failed executions remain failures; importing +a definition does not move its active schedule. + Generate the common series catalog after recording or completing an evaluation: ```sh diff --git a/migration/evidence.json b/migration/evidence.json index 69821c6c4..bcc20b61e 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -745,6 +745,102 @@ "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", "execution_qualification": "Scoped output and completion checks passed; one sample; publication unreviewed", "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37148375181, + "verified_files": 13, + "gaps": [], + "sha256": "49df16499aee7a038dfea048f8ec82e605cbfa22fa8822e0060bc7ce213e65b4", + "bundle_bytes": 372736, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148375181.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37148377350, + "verified_files": 13, + "gaps": [], + "sha256": "bf89e6b23b5517ca298ff620dde1885fce19ec9a9a8d92fce5092a2dfd93e893", + "bundle_bytes": 373248, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148377350.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37148378881, + "verified_files": 13, + "gaps": [], + "sha256": "de853e9d1c1b0b6bead7e1822a5efbc3a0bb92b15e37e1e53c400757b2354c56", + "bundle_bytes": 263168, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148378881.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37149053291, + "verified_files": 13, + "gaps": [], + "sha256": "899c7adc2f77d143cfdda44d59b9f12884877c5a5fcf65bed1c5989341c8b2a9", + "bundle_bytes": 266240, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149053291.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37149055130, + "verified_files": 13, + "gaps": [], + "sha256": "36d2b8fe86fbfe42fa9df155468b0beda3b6a1baabec398dd3a577e04feafafd", + "bundle_bytes": 1602560, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149055130.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37149056843, + "verified_files": 13, + "gaps": [], + "sha256": "b1b97061460c375759d37184868d53cddad9c1b980b4e69d8e6a67cce374ce09", + "bundle_bytes": 1658880, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149056843.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", + "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" } ] } diff --git a/results/chroma/image-output-qualification-01/interpretation.md b/results/chroma/image-output-qualification-01/interpretation.md index 3e2bf8806..1869d4bd7 100644 --- a/results/chroma/image-output-qualification-01/interpretation.md +++ b/results/chroma/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 4 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/chroma/image-output-qualification-01/methodology-review.json b/results/chroma/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..fccd872a6 --- /dev/null +++ b/results/chroma/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104573058" + ] +} diff --git a/results/chroma/image-output-qualification-01/report.json b/results/chroma/image-output-qualification-01/report.json index 99a4f1d19..a799ce080 100644 --- a/results/chroma/image-output-qualification-01/report.json +++ b/results/chroma/image-output-qualification-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "c7cadaea12f1ef9c03029ddfd676f4d17e8395dba5d53dd806d12ba12c8671e7", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/chroma/image-output-qualification-01/report.md b/results/chroma/image-output-qualification-01/report.md index d9ab6a439..aaa98dbec 100644 --- a/results/chroma/image-output-qualification-01/report.md +++ b/results/chroma/image-output-qualification-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 5097 | 5097–5097 | 2365446028 | 1 | | warm | boringcache | 1 | 31 | 31–31 | 2365446028 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/duckgres/image-output-qualification-01/interpretation.md b/results/duckgres/image-output-qualification-01/interpretation.md index 1aed6fd8f..57bed75a2 100644 --- a/results/duckgres/image-output-qualification-01/interpretation.md +++ b/results/duckgres/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 4 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/duckgres/image-output-qualification-01/methodology-review.json b/results/duckgres/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..877ee3280 --- /dev/null +++ b/results/duckgres/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104593175" + ] +} diff --git a/results/duckgres/image-output-qualification-01/report.json b/results/duckgres/image-output-qualification-01/report.json index 0f2f1e3e1..4fb956d58 100644 --- a/results/duckgres/image-output-qualification-01/report.json +++ b/results/duckgres/image-output-qualification-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "33a5d7375e15d5747d7e19e330c812f6c9690e95a62b4cdf71697173fdc7bd20", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/duckgres/image-output-qualification-01/report.md b/results/duckgres/image-output-qualification-01/report.md index 4f246054d..f90044b64 100644 --- a/results/duckgres/image-output-qualification-01/report.md +++ b/results/duckgres/image-output-qualification-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 193 | 193–193 | 1880170140 | 1 | | warm | boringcache | 1 | 20 | 20–20 | 1880170140 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/hugo/image-output-qualification-02/completions/37148378881.json b/results/hugo/image-output-qualification-02/completions/37148378881.json new file mode 100644 index 000000000..3de737c37 --- /dev/null +++ b/results/hugo/image-output-qualification-02/completions/37148378881.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148378881", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6560d3ffaa57bd746e57c74503987869c470c24ad4b4a02632b6392ad18239aa" +} diff --git a/results/hugo/image-output-qualification-02/interpretation.md b/results/hugo/image-output-qualification-02/interpretation.md new file mode 100644 index 000000000..bec5714f9 --- /dev/null +++ b/results/hugo/image-output-qualification-02/interpretation.md @@ -0,0 +1,7 @@ +# hugo — image-output-qualification-02 + +The declared one-sample cold and warm phases completed and verified their image outputs. The [generated report](report.md) retains all four observations and measured storage coverage. This screen does not establish a repeatable performance advantage. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. + +The original run, attempts, logs, and artifacts are retained in the [verified lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04). diff --git a/results/hugo/image-output-qualification-02/methodology-review.json b/results/hugo/image-output-qualification-02/methodology-review.json new file mode 100644 index 000000000..bae7c5f97 --- /dev/null +++ b/results/hugo/image-output-qualification-02/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] +} diff --git a/results/hugo/image-output-qualification-02/report.json b/results/hugo/image-output-qualification-02/report.json new file mode 100644 index 000000000..abfbcc993 --- /dev/null +++ b/results/hugo/image-output-qualification-02/report.json @@ -0,0 +1,836 @@ +{ + "schema_version": 1, + "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148378881", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6560d3ffaa57bd746e57c74503987869c470c24ad4b4a02632b6392ad18239aa" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 270, + "min": 270, + "max": 270 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 223, + "min": 223, + "max": 223 + }, + "storage": { + "median": 411514484, + "min": 411514484, + "max": 411514484 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": { + "median": 411514484, + "min": 411514484, + "max": 411514484 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 264, + "workflow_seconds": 272, + "total_seconds": 270, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 270 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-image-output-qualification-02-s1-r37148378881-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171227", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 18, + "workflow_seconds": 31, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-image-output-qualification-02-s1-r37148378881-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171229", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 223, + "workflow_seconds": 227, + "total_seconds": 223, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 223 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + }, + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411514484, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411514484, + "unmeasured_tags": [], + "total_bytes": 411514484, + "observations": [ + { + "requested_tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "96e3380c-67d8-44b4-a951-c1dc410c6bd4", + "manifest_root_digest": "sha256:709249e89b6c464bb7165b66f4dd9fb708780f3d74dc65d7456b59f21bf02446", + "kv_total_size": 0, + "compressed_size": 411514484, + "size": 411514484 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171226", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 28, + "workflow_seconds": 30, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": null + }, + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411514484, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411514484, + "unmeasured_tags": [], + "total_bytes": 411514484, + "observations": [ + { + "requested_tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "96e3380c-67d8-44b4-a951-c1dc410c6bd4", + "manifest_root_digest": "sha256:709249e89b6c464bb7165b66f4dd9fb708780f3d74dc65d7456b59f21bf02446", + "kv_total_size": 0, + "compressed_size": 411514484, + "size": 411514484 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171228", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] + } + ], + "exclusions": [] +} diff --git a/results/hugo/image-output-qualification-02/report.md b/results/hugo/image-output-qualification-02/report.md new file mode 100644 index 000000000..5f68e5783 --- /dev/null +++ b/results/hugo/image-output-qualification-02/report.md @@ -0,0 +1,25 @@ +# hugo: image-output-qualification-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 270 | 270–270 | unmeasured | 0 | +| warm | actions-cache | 1 | 28 | 28–28 | unmeasured | 0 | +| cold | boringcache | 1 | 223 | 223–223 | 411514484 | 1 | +| warm | boringcache | 1 | 28 | 28–28 | 411514484 | 1 | + +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/hugo/image-output-qualification-02/runs/1-actions-cache-cold.json b/results/hugo/image-output-qualification-02/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..7ee882adc --- /dev/null +++ b/results/hugo/image-output-qualification-02/runs/1-actions-cache-cold.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 264, + "workflow_seconds": 272, + "total_seconds": 270, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 270 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-image-output-qualification-02-s1-r37148378881-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171227", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] +} diff --git a/results/hugo/image-output-qualification-02/runs/1-actions-cache-warm.json b/results/hugo/image-output-qualification-02/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..4cd3085d6 --- /dev/null +++ b/results/hugo/image-output-qualification-02/runs/1-actions-cache-warm.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 18, + "workflow_seconds": 31, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-image-output-qualification-02-s1-r37148378881-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171229", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] +} diff --git a/results/hugo/image-output-qualification-02/runs/1-boringcache-cold.json b/results/hugo/image-output-qualification-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..8033e51f4 --- /dev/null +++ b/results/hugo/image-output-qualification-02/runs/1-boringcache-cold.json @@ -0,0 +1,208 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 223, + "workflow_seconds": 227, + "total_seconds": 223, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 223 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + }, + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411514484, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411514484, + "unmeasured_tags": [], + "total_bytes": 411514484, + "observations": [ + { + "requested_tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "96e3380c-67d8-44b4-a951-c1dc410c6bd4", + "manifest_root_digest": "sha256:709249e89b6c464bb7165b66f4dd9fb708780f3d74dc65d7456b59f21bf02446", + "kv_total_size": 0, + "compressed_size": 411514484, + "size": 411514484 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171226", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] +} diff --git a/results/hugo/image-output-qualification-02/runs/1-boringcache-warm.json b/results/hugo/image-output-qualification-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..22220fec1 --- /dev/null +++ b/results/hugo/image-output-qualification-02/runs/1-boringcache-warm.json @@ -0,0 +1,208 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 28, + "workflow_seconds": 30, + "total_seconds": 28, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 28 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": null + }, + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411514484, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411514484, + "unmeasured_tags": [], + "total_bytes": 411514484, + "observations": [ + { + "requested_tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "96e3380c-67d8-44b4-a951-c1dc410c6bd4", + "manifest_root_digest": "sha256:709249e89b6c464bb7165b66f4dd9fb708780f3d74dc65d7456b59f21bf02446", + "kv_total_size": 0, + "compressed_size": 411514484, + "size": 411514484 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-image-output-qualification-02-s1-r37148378881-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148378881", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171228", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148378881-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + ] +} diff --git a/results/immich/image-output-qualification-01/interpretation.md b/results/immich/image-output-qualification-01/interpretation.md index 040477161..efe3b8cd3 100644 --- a/results/immich/image-output-qualification-01/interpretation.md +++ b/results/immich/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 4 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/immich/image-output-qualification-01/methodology-review.json b/results/immich/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..3e5149292 --- /dev/null +++ b/results/immich/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + ] +} diff --git a/results/immich/image-output-qualification-01/report.json b/results/immich/image-output-qualification-01/report.json index f545bd75a..e6a0a369c 100644 --- a/results/immich/image-output-qualification-01/report.json +++ b/results/immich/image-output-qualification-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/immich/image-output-qualification-01/report.md b/results/immich/image-output-qualification-01/report.md index a806b9c7b..4e48dfb48 100644 --- a/results/immich/image-output-qualification-01/report.md +++ b/results/immich/image-output-qualification-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 207 | 207–207 | 1758920230 | 1 | | warm | boringcache | 1 | 38 | 38–38 | 1758920230 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/linkerd2/image-output-qualification-01/interpretation.md b/results/linkerd2/image-output-qualification-01/interpretation.md index afc528124..c1229955a 100644 --- a/results/linkerd2/image-output-qualification-01/interpretation.md +++ b/results/linkerd2/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 4 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/linkerd2/image-output-qualification-01/methodology-review.json b/results/linkerd2/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..fec5e6820 --- /dev/null +++ b/results/linkerd2/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + ] +} diff --git a/results/linkerd2/image-output-qualification-01/report.json b/results/linkerd2/image-output-qualification-01/report.json index 4ed7af69c..79e2d831c 100644 --- a/results/linkerd2/image-output-qualification-01/report.json +++ b/results/linkerd2/image-output-qualification-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/linkerd2/image-output-qualification-01/report.md b/results/linkerd2/image-output-qualification-01/report.md index 1f70b17dd..8bed2470e 100644 --- a/results/linkerd2/image-output-qualification-01/report.md +++ b/results/linkerd2/image-output-qualification-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 200 | 200–200 | 1230761384 | 1 | | warm | boringcache | 1 | 12 | 12–12 | 1230761384 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/mastodon/image-output-qualification-01/interpretation.md b/results/mastodon/image-output-qualification-01/interpretation.md index 586a137a7..c3f9c43e6 100644 --- a/results/mastodon/image-output-qualification-01/interpretation.md +++ b/results/mastodon/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 4 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/mastodon/image-output-qualification-01/methodology-review.json b/results/mastodon/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..b9738420e --- /dev/null +++ b/results/mastodon/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + ] +} diff --git a/results/mastodon/image-output-qualification-01/report.json b/results/mastodon/image-output-qualification-01/report.json index d9cba0286..e1b64ae8e 100644 --- a/results/mastodon/image-output-qualification-01/report.json +++ b/results/mastodon/image-output-qualification-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/mastodon/image-output-qualification-01/report.md b/results/mastodon/image-output-qualification-01/report.md index 02165ca14..d3bce6bf7 100644 --- a/results/mastodon/image-output-qualification-01/report.md +++ b/results/mastodon/image-output-qualification-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 424 | 424–424 | 1130445165 | 1 | | warm | boringcache | 1 | 38 | 38–38 | 1130445165 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/mastodon/streaming-output-qualification-01/completions/37149053291.json b/results/mastodon/streaming-output-qualification-01/completions/37149053291.json new file mode 100644 index 000000000..dc3141ff6 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/completions/37149053291.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", + "sample": 1, + "run_id": "37149053291", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "530bf111ae6b4a6f70347e2e293b0ad75cec21dd175728be464cf8ecfc1ebfec" +} diff --git a/results/mastodon/streaming-output-qualification-01/interpretation.md b/results/mastodon/streaming-output-qualification-01/interpretation.md new file mode 100644 index 000000000..38c4927f1 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/interpretation.md @@ -0,0 +1,7 @@ +# mastodon — streaming-output-qualification-01 + +The declared one-sample cold and warm phases completed and verified their image outputs. The [generated report](report.md) retains all four observations and measured storage coverage. This screen does not establish a repeatable performance advantage. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. + +The original run, attempts, logs, and artifacts are retained in the [verified lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04). diff --git a/results/mastodon/streaming-output-qualification-01/methodology-review.json b/results/mastodon/streaming-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..2d269478c --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] +} diff --git a/results/mastodon/streaming-output-qualification-01/report.json b/results/mastodon/streaming-output-qualification-01/report.json new file mode 100644 index 000000000..97f461bc1 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/report.json @@ -0,0 +1,978 @@ +{ + "schema_version": 1, + "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", + "sample": 1, + "run_id": "37149053291", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "530bf111ae6b4a6f70347e2e293b0ad75cec21dd175728be464cf8ecfc1ebfec" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 52, + "min": 52, + "max": 52 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 21, + "min": 21, + "max": 21 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 40, + "min": 40, + "max": 40 + }, + "storage": { + "median": 106281953, + "min": 106281953, + "max": 106281953 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 22, + "min": 22, + "max": 22 + }, + "storage": { + "median": 106281953, + "min": 106281953, + "max": 106281953 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 41, + "workflow_seconds": 59, + "total_seconds": 52, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 52 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171234", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 15, + "workflow_seconds": 27, + "total_seconds": 21, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 21 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171236", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 40, + "workflow_seconds": 46, + "total_seconds": 40, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 40 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + }, + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106281953, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106281953, + "unmeasured_tags": [], + "total_bytes": 106281953, + "observations": [ + { + "requested_tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "948fad83-622f-4095-a8ee-b22f0cdf8681", + "manifest_root_digest": "sha256:d4042da37347a8cb95607c82d0b4d9b8e0b53bbbcf89c8808dbfbb2713b77425", + "kv_total_size": 0, + "compressed_size": 106281953, + "size": 106281953 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171233", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] + }, + { + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 22, + "workflow_seconds": 27, + "total_seconds": 22, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 22 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106281953, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106281953, + "unmeasured_tags": [], + "total_bytes": 106281953, + "observations": [ + { + "requested_tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "948fad83-622f-4095-a8ee-b22f0cdf8681", + "manifest_root_digest": "sha256:d4042da37347a8cb95607c82d0b4d9b8e0b53bbbcf89c8808dbfbb2713b77425", + "kv_total_size": 0, + "compressed_size": 106281953, + "size": 106281953 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171235", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] + } + ], + "exclusions": [] +} diff --git a/results/mastodon/streaming-output-qualification-01/report.md b/results/mastodon/streaming-output-qualification-01/report.md new file mode 100644 index 000000000..d1f51d6b4 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/report.md @@ -0,0 +1,25 @@ +# mastodon: streaming-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 52 | 52–52 | unmeasured | 0 | +| warm | actions-cache | 1 | 21 | 21–21 | unmeasured | 0 | +| cold | boringcache | 1 | 40 | 40–40 | 106281953 | 1 | +| warm | boringcache | 1 | 22 | 22–22 | 106281953 | 1 | + +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-cold.json b/results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..551930d94 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 41, + "workflow_seconds": 59, + "total_seconds": 52, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 52 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171234", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] +} diff --git a/results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-warm.json b/results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..df05016a6 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 15, + "workflow_seconds": 27, + "total_seconds": 21, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 21 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171236", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] +} diff --git a/results/mastodon/streaming-output-qualification-01/runs/1-boringcache-cold.json b/results/mastodon/streaming-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..4a9180232 --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,244 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 40, + "workflow_seconds": 46, + "total_seconds": 40, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 40 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + }, + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106281953, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106281953, + "unmeasured_tags": [], + "total_bytes": 106281953, + "observations": [ + { + "requested_tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "948fad83-622f-4095-a8ee-b22f0cdf8681", + "manifest_root_digest": "sha256:d4042da37347a8cb95607c82d0b4d9b8e0b53bbbcf89c8808dbfbb2713b77425", + "kv_total_size": 0, + "compressed_size": 106281953, + "size": 106281953 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171233", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] +} diff --git a/results/mastodon/streaming-output-qualification-01/runs/1-boringcache-warm.json b/results/mastodon/streaming-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..ca1022cab --- /dev/null +++ b/results/mastodon/streaming-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,244 @@ +{ + "schema_version": 1, + "benchmark": "mastodon-streaming", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "streaming", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "mastodon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-mastodon", + "branch": "main", + "revision": "af806ebb3a531c4196b47bfc655da5448db5ba54" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "mastodon" + }, + "historical_workflow": "mastodon-mastodon-fresh-benchmark.yml", + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/mastodon-mastodon-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 15, + "cache_tags": { + "mastodon-docker-local": "{scope}-docker", + "mastodon-sccache-local": "{scope}-sccache" + }, + "native": { + "action": ".github/actions/mastodon-docker-benchmark", + "benchmark_id": "mastodon", + "fresh_inputs": { + "workload": "server", + "load_image": "true" + }, + "rolling_inputs": { + "workload": "server" + }, + "variants": { + "server": {}, + "server-sccache": { + "benchmark_id": "mastodon-sccache", + "fresh_inputs": { + "docker_tool_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false" + } + } + }, + "streaming": { + "benchmark_id": "mastodon-streaming", + "fresh_inputs": { + "workload": "streaming" + }, + "rolling_inputs": { + "workload": "streaming" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 22, + "workflow_seconds": 27, + "total_seconds": 22, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 22 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 106281953, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 106281953, + "unmeasured_tags": [], + "total_bytes": 106281953, + "observations": [ + { + "requested_tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "tag": "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "948fad83-622f-4095-a8ee-b22f0cdf8681", + "manifest_root_digest": "sha256:d4042da37347a8cb95607c82d0b4d9b8e0b53bbbcf89c8808dbfbb2713b77425", + "kv_total_size": 0, + "compressed_size": 106281953, + "size": 106281953 + } + ] + } + }, + "source": { + "repository": "mastodon/mastodon", + "sha": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "mastodon-streaming-series-streaming-output-qualification-01-s1-r37149053291-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149053291", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171235", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149053291-1", + "series": { + "id": "streaming-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + ] +} diff --git a/results/n8n/distroless-output-qualification-01/completions/37149056843.json b/results/n8n/distroless-output-qualification-01/completions/37149056843.json new file mode 100644 index 000000000..f9541ca6c --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/completions/37149056843.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", + "sample": 1, + "run_id": "37149056843", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4bbc22e12b0a3e5e9c1c5c481f68f29002291e60a1bcc748ea3e18eaaed8566e" +} diff --git a/results/n8n/distroless-output-qualification-01/interpretation.md b/results/n8n/distroless-output-qualification-01/interpretation.md new file mode 100644 index 000000000..bd3c8f48a --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/interpretation.md @@ -0,0 +1,7 @@ +# n8n — distroless-output-qualification-01 + +The declared one-sample cold and warm phases completed and verified their image outputs. The [generated report](report.md) retains all four observations and measured storage coverage. This screen does not establish a repeatable performance advantage. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. + +The original run, attempts, logs, and artifacts are retained in the [verified lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04). diff --git a/results/n8n/distroless-output-qualification-01/methodology-review.json b/results/n8n/distroless-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..2a465afe4 --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] +} diff --git a/results/n8n/distroless-output-qualification-01/report.json b/results/n8n/distroless-output-qualification-01/report.json new file mode 100644 index 000000000..339630392 --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/report.json @@ -0,0 +1,1082 @@ +{ + "schema_version": 1, + "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", + "sample": 1, + "run_id": "37149056843", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4bbc22e12b0a3e5e9c1c5c481f68f29002291e60a1bcc748ea3e18eaaed8566e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 214, + "min": 214, + "max": 214 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 132, + "min": 132, + "max": 132 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 81, + "min": 81, + "max": 81 + }, + "storage": { + "median": 581055738, + "min": 581055738, + "max": 581055738 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 110, + "min": 110, + "max": 110 + }, + "storage": { + "median": 581055738, + "min": 581055738, + "max": 581055738 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 205, + "workflow_seconds": 423, + "total_seconds": 214, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 214 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171251", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 124, + "workflow_seconds": 295, + "total_seconds": 132, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 132 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171254", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 81, + "workflow_seconds": 283, + "total_seconds": 81, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 81 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581055738, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581055738, + "unmeasured_tags": [], + "total_bytes": 581055738, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "23a7d39e-2428-4b17-b2dd-9317274b02ec", + "manifest_root_digest": "sha256:bb4c11a6ca0dca3b18e3686ab959f3d247e4436c253f15836439ebd6e3c27a13", + "kv_total_size": 0, + "compressed_size": 581055738, + "size": 581055738 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171250", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 110, + "workflow_seconds": 314, + "total_seconds": 110, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 110 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581055738, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581055738, + "unmeasured_tags": [], + "total_bytes": 581055738, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "23a7d39e-2428-4b17-b2dd-9317274b02ec", + "manifest_root_digest": "sha256:bb4c11a6ca0dca3b18e3686ab959f3d247e4436c253f15836439ebd6e3c27a13", + "kv_total_size": 0, + "compressed_size": 581055738, + "size": 581055738 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171255", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/distroless-output-qualification-01/report.md b/results/n8n/distroless-output-qualification-01/report.md new file mode 100644 index 000000000..efd7e1ed4 --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/report.md @@ -0,0 +1,25 @@ +# n8n: distroless-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 214 | 214–214 | unmeasured | 0 | +| warm | actions-cache | 1 | 132 | 132–132 | unmeasured | 0 | +| cold | boringcache | 1 | 81 | 81–81 | 581055738 | 1 | +| warm | boringcache | 1 | 110 | 110–110 | 581055738 | 1 | + +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/n8n/distroless-output-qualification-01/runs/1-actions-cache-cold.json b/results/n8n/distroless-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..669b5c067 --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 205, + "workflow_seconds": 423, + "total_seconds": 214, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 214 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171251", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] +} diff --git a/results/n8n/distroless-output-qualification-01/runs/1-actions-cache-warm.json b/results/n8n/distroless-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..7e688a127 --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 124, + "workflow_seconds": 295, + "total_seconds": 132, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 132 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171254", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] +} diff --git a/results/n8n/distroless-output-qualification-01/runs/1-boringcache-cold.json b/results/n8n/distroless-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..32b778411 --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 81, + "workflow_seconds": 283, + "total_seconds": 81, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 81 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + }, + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581055738, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581055738, + "unmeasured_tags": [], + "total_bytes": 581055738, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "23a7d39e-2428-4b17-b2dd-9317274b02ec", + "manifest_root_digest": "sha256:bb4c11a6ca0dca3b18e3686ab959f3d247e4436c253f15836439ebd6e3c27a13", + "kv_total_size": 0, + "compressed_size": 581055738, + "size": 581055738 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171250", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] +} diff --git a/results/n8n/distroless-output-qualification-01/runs/1-boringcache-warm.json b/results/n8n/distroless-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..c5ec2309b --- /dev/null +++ b/results/n8n/distroless-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners-distroless", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "distroless", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 110, + "workflow_seconds": 314, + "total_seconds": 110, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 110 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 581055738, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 581055738, + "unmeasured_tags": [], + "total_bytes": 581055738, + "observations": [ + { + "requested_tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "tag": "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "23a7d39e-2428-4b17-b2dd-9317274b02ec", + "manifest_root_digest": "sha256:bb4c11a6ca0dca3b18e3686ab959f3d247e4436c253f15836439ebd6e3c27a13", + "kv_total_size": 0, + "compressed_size": 581055738, + "size": 581055738 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-distroless-series-distroless-output-qualification-01-s1-r37149056843-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149056843", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171255", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149056843-1", + "series": { + "id": "distroless-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + ] +} diff --git a/results/n8n/image-output-qualification-01/interpretation.md b/results/n8n/image-output-qualification-01/interpretation.md index 1993fbdb3..828527420 100644 --- a/results/n8n/image-output-qualification-01/interpretation.md +++ b/results/n8n/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 4 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/n8n/image-output-qualification-01/methodology-review.json b/results/n8n/image-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..c51aa4f12 --- /dev/null +++ b/results/n8n/image-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + ] +} diff --git a/results/n8n/image-output-qualification-01/report.json b/results/n8n/image-output-qualification-01/report.json index f0d3dfff6..1bc8e6951 100644 --- a/results/n8n/image-output-qualification-01/report.json +++ b/results/n8n/image-output-qualification-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/n8n/image-output-qualification-01/report.md b/results/n8n/image-output-qualification-01/report.md index 2bbb0a0f6..51bee1749 100644 --- a/results/n8n/image-output-qualification-01/report.md +++ b/results/n8n/image-output-qualification-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 319 | 319–319 | 483433384 | 1 | | warm | boringcache | 1 | 162 | 162–162 | 483433384 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/n8n/runners-output-qualification-01/completions/37149055130.json b/results/n8n/runners-output-qualification-01/completions/37149055130.json new file mode 100644 index 000000000..a3a72d97c --- /dev/null +++ b/results/n8n/runners-output-qualification-01/completions/37149055130.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "n8n", + "series_id": "runners-output-qualification-01", + "sample": 1, + "run_id": "37149055130", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c52e6d5bedd2e1d5e4333c820e0895b30eee8eb5c5ec02389cb5d669dc6f7017" +} diff --git a/results/n8n/runners-output-qualification-01/interpretation.md b/results/n8n/runners-output-qualification-01/interpretation.md new file mode 100644 index 000000000..518c2b420 --- /dev/null +++ b/results/n8n/runners-output-qualification-01/interpretation.md @@ -0,0 +1,7 @@ +# n8n — runners-output-qualification-01 + +The declared one-sample cold and warm phases completed and verified their image outputs. The [generated report](report.md) retains all four observations and measured storage coverage. This screen does not establish a repeatable performance advantage. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. + +The original run, attempts, logs, and artifacts are retained in the [verified lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04). diff --git a/results/n8n/runners-output-qualification-01/methodology-review.json b/results/n8n/runners-output-qualification-01/methodology-review.json new file mode 100644 index 000000000..8a1b6e9e9 --- /dev/null +++ b/results/n8n/runners-output-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] +} diff --git a/results/n8n/runners-output-qualification-01/report.json b/results/n8n/runners-output-qualification-01/report.json new file mode 100644 index 000000000..c21ecbdba --- /dev/null +++ b/results/n8n/runners-output-qualification-01/report.json @@ -0,0 +1,1082 @@ +{ + "schema_version": 1, + "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "n8n", + "series_id": "runners-output-qualification-01", + "sample": 1, + "run_id": "37149055130", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c52e6d5bedd2e1d5e4333c820e0895b30eee8eb5c5ec02389cb5d669dc6f7017" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 144, + "min": 144, + "max": 144 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 89, + "min": 89, + "max": 89 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 62, + "min": 62, + "max": 62 + }, + "storage": { + "median": 413827357, + "min": 413827357, + "max": 413827357 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 60, + "min": 60, + "max": 60 + }, + "storage": { + "median": 413827357, + "min": 413827357, + "max": 413827357 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 136, + "workflow_seconds": 340, + "total_seconds": 144, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 144 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171239", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 78, + "workflow_seconds": 268, + "total_seconds": 89, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 89 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171247", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 62, + "workflow_seconds": 239, + "total_seconds": 62, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 62 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + }, + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413827357, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413827357, + "unmeasured_tags": [], + "total_bytes": 413827357, + "observations": [ + { + "requested_tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "8ac825cd-7879-4adc-889b-f705bc27b0af", + "manifest_root_digest": "sha256:13d9e504ed366759c22be0e96114094eff72af357b0a0787619fe3d88d5f7abb", + "kv_total_size": 0, + "compressed_size": 413827357, + "size": 413827357 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171238", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] + }, + { + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 60, + "workflow_seconds": 256, + "total_seconds": 60, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 60 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413827357, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413827357, + "unmeasured_tags": [], + "total_bytes": 413827357, + "observations": [ + { + "requested_tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "8ac825cd-7879-4adc-889b-f705bc27b0af", + "manifest_root_digest": "sha256:13d9e504ed366759c22be0e96114094eff72af357b0a0787619fe3d88d5f7abb", + "kv_total_size": 0, + "compressed_size": 413827357, + "size": 413827357 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171246", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] + } + ], + "exclusions": [] +} diff --git a/results/n8n/runners-output-qualification-01/report.md b/results/n8n/runners-output-qualification-01/report.md new file mode 100644 index 000000000..51c6c75f0 --- /dev/null +++ b/results/n8n/runners-output-qualification-01/report.md @@ -0,0 +1,25 @@ +# n8n: runners-output-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 144 | 144–144 | unmeasured | 0 | +| warm | actions-cache | 1 | 89 | 89–89 | unmeasured | 0 | +| cold | boringcache | 1 | 62 | 62–62 | 413827357 | 1 | +| warm | boringcache | 1 | 60 | 60–60 | 413827357 | 1 | + +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/n8n/runners-output-qualification-01/runs/1-actions-cache-cold.json b/results/n8n/runners-output-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..ad63f70ef --- /dev/null +++ b/results/n8n/runners-output-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 136, + "workflow_seconds": 340, + "total_seconds": 144, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 144 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171239", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] +} diff --git a/results/n8n/runners-output-qualification-01/runs/1-actions-cache-warm.json b/results/n8n/runners-output-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..97463db9c --- /dev/null +++ b/results/n8n/runners-output-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 11, + "build_seconds": 78, + "workflow_seconds": 268, + "total_seconds": 89, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 89 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171247", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] +} diff --git a/results/n8n/runners-output-qualification-01/runs/1-boringcache-cold.json b/results/n8n/runners-output-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..36004793c --- /dev/null +++ b/results/n8n/runners-output-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 62, + "workflow_seconds": 239, + "total_seconds": 62, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 62 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + }, + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413827357, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413827357, + "unmeasured_tags": [], + "total_bytes": 413827357, + "observations": [ + { + "requested_tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "8ac825cd-7879-4adc-889b-f705bc27b0af", + "manifest_root_digest": "sha256:13d9e504ed366759c22be0e96114094eff72af357b0a0787619fe3d88d5f7abb", + "kv_total_size": 0, + "compressed_size": 413827357, + "size": 413827357 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171238", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] +} diff --git a/results/n8n/runners-output-qualification-01/runs/1-boringcache-warm.json b/results/n8n/runners-output-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..4c61af386 --- /dev/null +++ b/results/n8n/runners-output-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "benchmark": "n8n-runners", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "runners", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "n8n", + "workspace": "boringcache/benchmarks", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "source": { + "repository": "n8n-io/n8n", + "pins": [ + { + "path": "upstream", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-n8n", + "branch": "main", + "revision": "dd69b0de722c645f9b4719556e390446010f7fcc" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "n8n" + }, + "historical_workflow": "n8n-n8n-docker-fresh-benchmark.yml", + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/n8n-n8n-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-docker-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/n8n-n8n-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 17, + "cache_tags": { + "n8n-docker": "{scope}-docker", + "n8n-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/n8n-turbo-benchmark", + "benchmark_id": "n8n", + "fresh_inputs": { + "node_version": "26.7.0" + }, + "rolling_inputs": { + "node_version": "26.7.0" + }, + "variants": { + "turbo": {}, + "docker": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-docker", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", + "image": "n8n-benchmark" + } + }, + "runners": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile", + "image": "n8n-runners-benchmark" + } + }, + "distroless": { + "action": ".github/actions/n8n-docker-benchmark", + "benchmark_id": "n8n-runners-distroless", + "fresh_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" + }, + "rolling_inputs": { + "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", + "image": "n8n-runners-distroless-benchmark" + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 60, + "workflow_seconds": 256, + "total_seconds": 60, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", + "build_and_reuse_seconds": 60 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 413827357, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 413827357, + "unmeasured_tags": [], + "total_bytes": 413827357, + "observations": [ + { + "requested_tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "tag": "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "8ac825cd-7879-4adc-889b-f705bc27b0af", + "manifest_root_digest": "sha256:13d9e504ed366759c22be0e96114094eff72af357b0a0787619fe3d88d5f7abb", + "kv_total_size": 0, + "compressed_size": 413827357, + "size": 413827357 + } + ] + } + }, + "source": { + "repository": "n8n-io/n8n", + "sha": "7576de5f26ccfd20c1aceee0e6f5657833e267d9" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "n8n-runners-series-runners-output-qualification-01-s1-r37149055130-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37149055130", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "active-variant-qualification-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171246", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37149055130-1", + "series": { + "id": "runners-output-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Turbo variant: built CLI entrypoint exists; Docker variants: freshly loaded output image exists" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + ] +} diff --git a/results/posthog/image-output-combined-qualification-01/completions/37148377350.json b/results/posthog/image-output-combined-qualification-01/completions/37148377350.json new file mode 100644 index 000000000..75aafd83d --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/completions/37148377350.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", + "sample": 1, + "run_id": "37148377350", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7d2cf7f6e9b12c476c9b3e23b73d31fbe8f25176a91bf7d18fc29a027e819895" +} diff --git a/results/posthog/image-output-combined-qualification-01/interpretation.md b/results/posthog/image-output-combined-qualification-01/interpretation.md new file mode 100644 index 000000000..afcfb8ad8 --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/interpretation.md @@ -0,0 +1,7 @@ +# posthog — image-output-combined-qualification-01 + +The declared one-sample cold and warm phases completed and verified their image outputs. The [generated report](report.md) retains all four observations and measured storage coverage. This screen does not establish a repeatable performance advantage. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. + +The original run, attempts, logs, and artifacts are retained in the [verified lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04). diff --git a/results/posthog/image-output-combined-qualification-01/methodology-review.json b/results/posthog/image-output-combined-qualification-01/methodology-review.json new file mode 100644 index 000000000..c0b5d550c --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] +} diff --git a/results/posthog/image-output-combined-qualification-01/report.json b/results/posthog/image-output-combined-qualification-01/report.json new file mode 100644 index 000000000..d5f9adcbd --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/report.json @@ -0,0 +1,950 @@ +{ + "schema_version": 1, + "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", + "sample": 1, + "run_id": "37148377350", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7d2cf7f6e9b12c476c9b3e23b73d31fbe8f25176a91bf7d18fc29a027e819895" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1889, + "min": 1889, + "max": 1889 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 187, + "min": 187, + "max": 187 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 1016, + "min": 1016, + "max": 1016 + }, + "storage": { + "median": 6237909312, + "min": 6237909312, + "max": 6237909312 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 162, + "min": 162, + "max": 162 + }, + "storage": { + "median": 6237909312, + "min": 6237909312, + "max": 6237909312 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 1881, + "workflow_seconds": 1922, + "total_seconds": 1889, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1889 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171199", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 14, + "build_seconds": 173, + "workflow_seconds": 209, + "total_seconds": 187, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 187 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171223", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 1016, + "workflow_seconds": 1047, + "total_seconds": 1016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1016 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + }, + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237909312, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237909312, + "unmeasured_tags": [], + "total_bytes": 6237909312, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e73cf83a-3b44-46f0-86e5-a7b5eed68e49", + "manifest_root_digest": "sha256:53d2e811380722bbd461d9a693d2e91ceb5e492d8ffcf1eeb707a4ed0a7e0663", + "kv_total_size": 0, + "compressed_size": 6237909312, + "size": 6237909312 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171198", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 162, + "workflow_seconds": 183, + "total_seconds": 162, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 162 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237909312, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237909312, + "unmeasured_tags": [], + "total_bytes": 6237909312, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e73cf83a-3b44-46f0-86e5-a7b5eed68e49", + "manifest_root_digest": "sha256:53d2e811380722bbd461d9a693d2e91ceb5e492d8ffcf1eeb707a4ed0a7e0663", + "kv_total_size": 0, + "compressed_size": 6237909312, + "size": 6237909312 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171224", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/image-output-combined-qualification-01/report.md b/results/posthog/image-output-combined-qualification-01/report.md new file mode 100644 index 000000000..aee0d4b37 --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/report.md @@ -0,0 +1,25 @@ +# posthog: image-output-combined-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 1889 | 1889–1889 | unmeasured | 0 | +| warm | actions-cache | 1 | 187 | 187–187 | unmeasured | 0 | +| cold | boringcache | 1 | 1016 | 1016–1016 | 6237909312 | 1 | +| warm | boringcache | 1 | 162 | 162–162 | 6237909312 | 1 | + +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-cold.json b/results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..168937f07 --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 8, + "build_seconds": 1881, + "workflow_seconds": 1922, + "total_seconds": 1889, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1889 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171199", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] +} diff --git a/results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-warm.json b/results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..cd2d8899e --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 14, + "build_seconds": 173, + "workflow_seconds": 209, + "total_seconds": 187, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 187 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171223", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] +} diff --git a/results/posthog/image-output-combined-qualification-01/runs/1-boringcache-cold.json b/results/posthog/image-output-combined-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..347e7d304 --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 1016, + "workflow_seconds": 1047, + "total_seconds": 1016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1016 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + }, + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237909312, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237909312, + "unmeasured_tags": [], + "total_bytes": 6237909312, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e73cf83a-3b44-46f0-86e5-a7b5eed68e49", + "manifest_root_digest": "sha256:53d2e811380722bbd461d9a693d2e91ceb5e492d8ffcf1eeb707a4ed0a7e0663", + "kv_total_size": 0, + "compressed_size": 6237909312, + "size": 6237909312 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171198", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] +} diff --git a/results/posthog/image-output-combined-qualification-01/runs/1-boringcache-warm.json b/results/posthog/image-output-combined-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..4541737e6 --- /dev/null +++ b/results/posthog/image-output-combined-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 162, + "workflow_seconds": 183, + "total_seconds": 162, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 162 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237909312, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237909312, + "unmeasured_tags": [], + "total_bytes": 6237909312, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "e73cf83a-3b44-46f0-86e5-a7b5eed68e49", + "manifest_root_digest": "sha256:53d2e811380722bbd461d9a693d2e91ceb5e492d8ffcf1eeb707a4ed0a7e0663", + "kv_total_size": 0, + "compressed_size": 6237909312, + "size": 6237909312 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-image-output-combined-qualification-01-s1-r37148377350-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148377350", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171224", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148377350-1", + "series": { + "id": "image-output-combined-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + ] +} diff --git a/results/posthog/image-output-qualification-01/interpretation.md b/results/posthog/image-output-qualification-01/interpretation.md index f5209ad7b..e376a405f 100644 --- a/results/posthog/image-output-qualification-01/interpretation.md +++ b/results/posthog/image-output-qualification-01/interpretation.md @@ -9,3 +9,5 @@ The [generated report](report.md) retains 2 original phase records. Completion i Provider-reported storage stays labelled by its source. Missing measurements remain unmeasured; no cross-provider total storage saving is established. Repository-wide cache snapshots do not prove per-phase occupancy or absence of eviction. Evidence preservation is scoped to this run. It does not qualify every variant, caller cutover, website publication, or repository deletion. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/posthog/image-output-qualification-01/methodology-review.json b/results/posthog/image-output-qualification-01/methodology-review.json index 491c1a0ea..c14f2f3fe 100644 --- a/results/posthog/image-output-qualification-01/methodology-review.json +++ b/results/posthog/image-output-qualification-01/methodology-review.json @@ -3,7 +3,9 @@ "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", "issues": [ "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", - "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series.", + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." ], "evidence_links": [ "https://github.com/boringcache/benchmarks/actions/runs/37104600856" diff --git a/results/posthog/image-output-qualification-01/report.json b/results/posthog/image-output-qualification-01/report.json index 7e8b8e81f..a446f2cba 100644 --- a/results/posthog/image-output-qualification-01/report.json +++ b/results/posthog/image-output-qualification-01/report.json @@ -6,7 +6,9 @@ "failures": [], "methodology_issues": [ "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", - "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series." + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series.", + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." ], "execution_verified": false, "missing_completions": [], diff --git a/results/posthog/image-output-qualification-01/report.md b/results/posthog/image-output-qualification-01/report.md index ce8f27746..0d2962773 100644 --- a/results/posthog/image-output-qualification-01/report.md +++ b/results/posthog/image-output-qualification-01/report.md @@ -19,6 +19,8 @@ Methodology prevents a comparative claim: - Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution. - Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series. +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. Run 37104600856 failed completion checks: diff --git a/results/posthog/image-output-qualification-02/completions/37148375181.json b/results/posthog/image-output-qualification-02/completions/37148375181.json new file mode 100644 index 000000000..9e8184740 --- /dev/null +++ b/results/posthog/image-output-qualification-02/completions/37148375181.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148375181", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ad47456c459688cb932e9d232c57790b64b694616f587246c3bb3b3c07e77e25" +} diff --git a/results/posthog/image-output-qualification-02/interpretation.md b/results/posthog/image-output-qualification-02/interpretation.md new file mode 100644 index 000000000..e18ccbdb1 --- /dev/null +++ b/results/posthog/image-output-qualification-02/interpretation.md @@ -0,0 +1,7 @@ +# posthog — image-output-qualification-02 + +The declared one-sample cold and warm phases completed and verified their image outputs. The [generated report](report.md) retains all four observations and measured storage coverage. This screen does not establish a repeatable performance advantage. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. + +The original run, attempts, logs, and artifacts are retained in the [verified lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04). diff --git a/results/posthog/image-output-qualification-02/methodology-review.json b/results/posthog/image-output-qualification-02/methodology-review.json new file mode 100644 index 000000000..96ca45049 --- /dev/null +++ b/results/posthog/image-output-qualification-02/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] +} diff --git a/results/posthog/image-output-qualification-02/report.json b/results/posthog/image-output-qualification-02/report.json new file mode 100644 index 000000000..7542df6fa --- /dev/null +++ b/results/posthog/image-output-qualification-02/report.json @@ -0,0 +1,950 @@ +{ + "schema_version": 1, + "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148375181", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ad47456c459688cb932e9d232c57790b64b694616f587246c3bb3b3c07e77e25" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1609, + "min": 1609, + "max": 1609 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 175, + "min": 175, + "max": 175 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 778, + "min": 778, + "max": 778 + }, + "storage": { + "median": 6237262375, + "min": 6237262375, + "max": 6237262375 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 136, + "min": 136, + "max": 136 + }, + "storage": { + "median": 6237262375, + "min": 6237262375, + "max": 6237262375 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 13, + "build_seconds": 1596, + "workflow_seconds": 1642, + "total_seconds": 1609, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1609 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-image-output-qualification-02-s1-r37148375181-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171151", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 169, + "workflow_seconds": 207, + "total_seconds": 175, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 175 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-image-output-qualification-02-s1-r37148375181-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171196", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 778, + "workflow_seconds": 811, + "total_seconds": 778, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 778 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + }, + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237262375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237262375, + "unmeasured_tags": [], + "total_bytes": 6237262375, + "observations": [ + { + "requested_tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "ba59f4fc-5291-4e31-858e-434c21549334", + "manifest_root_digest": "sha256:9b36bdf48230f4fccfccf463c90764fbde739f73dd9868801c10f34fc1579e8f", + "kv_total_size": 0, + "compressed_size": 6237262375, + "size": 6237262375 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171150", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 136, + "workflow_seconds": 168, + "total_seconds": 136, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 136 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237262375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237262375, + "unmeasured_tags": [], + "total_bytes": 6237262375, + "observations": [ + { + "requested_tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "ba59f4fc-5291-4e31-858e-434c21549334", + "manifest_root_digest": "sha256:9b36bdf48230f4fccfccf463c90764fbde739f73dd9868801c10f34fc1579e8f", + "kv_total_size": 0, + "compressed_size": 6237262375, + "size": 6237262375 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171195", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/image-output-qualification-02/report.md b/results/posthog/image-output-qualification-02/report.md new file mode 100644 index 000000000..c358db712 --- /dev/null +++ b/results/posthog/image-output-qualification-02/report.md @@ -0,0 +1,25 @@ +# posthog: image-output-qualification-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 1609 | 1609–1609 | unmeasured | 0 | +| warm | actions-cache | 1 | 175 | 175–175 | unmeasured | 0 | +| cold | boringcache | 1 | 778 | 778–778 | 6237262375 | 1 | +| warm | boringcache | 1 | 136 | 136–136 | 6237262375 | 1 | + +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/image-output-qualification-02/runs/1-actions-cache-cold.json b/results/posthog/image-output-qualification-02/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..b6c50eba7 --- /dev/null +++ b/results/posthog/image-output-qualification-02/runs/1-actions-cache-cold.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 13, + "build_seconds": 1596, + "workflow_seconds": 1642, + "total_seconds": 1609, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1609 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-image-output-qualification-02-s1-r37148375181-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171151", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] +} diff --git a/results/posthog/image-output-qualification-02/runs/1-actions-cache-warm.json b/results/posthog/image-output-qualification-02/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..b071af583 --- /dev/null +++ b/results/posthog/image-output-qualification-02/runs/1-actions-cache-warm.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 6, + "build_seconds": 169, + "workflow_seconds": 207, + "total_seconds": 175, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 175 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-image-output-qualification-02-s1-r37148375181-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171196", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] +} diff --git a/results/posthog/image-output-qualification-02/runs/1-boringcache-cold.json b/results/posthog/image-output-qualification-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..6315fafeb --- /dev/null +++ b/results/posthog/image-output-qualification-02/runs/1-boringcache-cold.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 778, + "workflow_seconds": 811, + "total_seconds": 778, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 778 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + }, + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237262375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237262375, + "unmeasured_tags": [], + "total_bytes": 6237262375, + "observations": [ + { + "requested_tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "ba59f4fc-5291-4e31-858e-434c21549334", + "manifest_root_digest": "sha256:9b36bdf48230f4fccfccf463c90764fbde739f73dd9868801c10f34fc1579e8f", + "kv_total_size": 0, + "compressed_size": 6237262375, + "size": 6237262375 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171150", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] +} diff --git a/results/posthog/image-output-qualification-02/runs/1-boringcache-warm.json b/results/posthog/image-output-qualification-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..3c3c6907b --- /dev/null +++ b/results/posthog/image-output-qualification-02/runs/1-boringcache-warm.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 136, + "workflow_seconds": 168, + "total_seconds": 136, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 136 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6237262375, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6237262375, + "unmeasured_tags": [], + "total_bytes": 6237262375, + "observations": [ + { + "requested_tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "tag": "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "ba59f4fc-5291-4e31-858e-434c21549334", + "manifest_root_digest": "sha256:9b36bdf48230f4fccfccf463c90764fbde739f73dd9868801c10f34fc1579e8f", + "kv_total_size": 0, + "compressed_size": 6237262375, + "size": 6237262375 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-image-output-qualification-02-s1-r37148375181-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37148375181", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "docker-output-correction-2026-10-03", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000171195", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37148375181-1", + "series": { + "id": "image-output-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + ] +} diff --git a/results/qdrant/migration-screening-01/interpretation.md b/results/qdrant/migration-screening-01/interpretation.md index 7a5709108..df660da63 100644 --- a/results/qdrant/migration-screening-01/interpretation.md +++ b/results/qdrant/migration-screening-01/interpretation.md @@ -23,3 +23,5 @@ The [durable evidence](evidence.md) was independently downloaded and verified. Publication remains unreviewed. Image inspection does not test application runtime behavior. Rolling source changes, source advancement, schedules, and canary/release caller cutover require separate qualification. + +The October 4 methodology review found unequal warm cache publication: Actions Cache exported state while BoringCache restored without publishing. The original final One evidence envelope was not retained. These observations remain useful output/execution evidence, but cannot support a comparative claim. See [the plan-bound review](methodology-review.json). A corrected execution definition requires a new series. diff --git a/results/qdrant/migration-screening-01/methodology-review.json b/results/qdrant/migration-screening-01/methodology-review.json new file mode 100644 index 000000000..64699b3e7 --- /dev/null +++ b/results/qdrant/migration-screening-01/methodology-review.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "plan_sha256": "56f373789edbd2cc069a8e6feed14f4fca492f1c1381fe86ff3f20152050571f", + "issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37085343088" + ] +} diff --git a/results/qdrant/migration-screening-01/report.json b/results/qdrant/migration-screening-01/report.json index c63468512..a2c47a39e 100644 --- a/results/qdrant/migration-screening-01/report.json +++ b/results/qdrant/migration-screening-01/report.json @@ -2,9 +2,12 @@ "schema_version": 1, "plan_sha256": "56f373789edbd2cc069a8e6feed14f4fca492f1c1381fe86ff3f20152050571f", "complete": true, - "valid_for_comparison": true, + "valid_for_comparison": false, "failures": [], - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "execution_verified": true, "missing_completions": [], "failed_completions": [], diff --git a/results/qdrant/migration-screening-01/report.md b/results/qdrant/migration-screening-01/report.md index 52f9b7448..6fc89e461 100644 --- a/results/qdrant/migration-screening-01/report.md +++ b/results/qdrant/migration-screening-01/report.md @@ -17,4 +17,9 @@ Queue time, dependency setup outside the declared scope, and job duration are co | cold | boringcache | 1 | 576 | 576–576 | 2084676721 | 1 | | warm | boringcache | 1 | 25 | 25–25 | 2084676721 | 1 | +Methodology prevents a comparative claim: + +- The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle. +- The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup. + Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index 76bd33bb7..a0d0e872a 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -22,7 +22,7 @@ module BenchmarkCases ROOT = File.expand_path("..", __dir__) WORKSPACE = "boringcache/benchmarks" REPOSITORY = "boringcache/benchmarks" - HELPERS = %w[benchmark-plan run-benchmark-plan activate-docker-plan verify-docker-output summarize-cargo-evidence summarize-sccache-errors docker-case-contract measure-build native-case prepare-source scope-case-cache].freeze + HELPERS = %w[benchmark-plan benchmark-phase run-benchmark-plan activate-docker-plan verify-docker-output summarize-cargo-evidence summarize-sccache-errors docker-case-contract measure-build native-case prepare-source scope-case-cache].freeze class Error < StandardError; end def self.command(*args, chdir: nil, stdin: "", env: {}) @@ -44,7 +44,7 @@ def self.definition_sha256(item, root: ROOT) files += item.dig("execution", "workflows").map { |entry| File.join(root, entry.fetch("path")) } files += HELPERS.map { |name| File.join(root, "scripts", "#{name}.rb") } files += %w[scripts/canonical/benchmark-report.rb scripts/verify-upstream-recipe.rb scripts/benchmark-cases.rb scripts/benchmark-series.rb bin/bench Gemfile.lock .tool-versions].map { |path| File.join(root, path) } - files += Dir.glob(File.join(root, ".github", "actions", "**", "*"), File::FNM_DOTMATCH).select { |path| File.file?(path) } + files += shared_action_files(root: root) files += Dir.glob(File.join(root, "adapters", "docker", "**", "*"), File::FNM_DOTMATCH).select { |path| File.file?(path) } if item["adapter"] == "docker" manifest = files.uniq.sort.to_h { |path| [path.delete_prefix(root + "/"), Digest::SHA256.file(path).hexdigest] } Digest::SHA256.hexdigest(JSON.generate({"case" => item, "files" => manifest})) @@ -431,15 +431,28 @@ def self.contract_views(root: ROOT) end def self.copy_shared_actions(target, root: ROOT) - FileUtils.mkdir_p(File.join(target, ".github", "actions")) - FileUtils.cp_r(Dir[File.join(root, ".github", "actions", "*")], File.join(target, ".github", "actions")) + shared_action_files(root: root).each do |source| + destination = File.join(target, source.delete_prefix("#{root}/")) + FileUtils.mkdir_p(File.dirname(destination)) + FileUtils.cp(source, destination) + end + end + + def self.shared_action_files(root: ROOT) + require "find" + directory = File.join(root, ".github", "actions") + return [] unless Dir.exist?(directory) + Find.find(directory).filter_map do |path| + Find.prune if File.directory?(path) && File.basename(path) == "node_modules" + path if File.file?(path) + end end def self.resolve_provider_steps(document, provider) case document when Hash if document["uses"] == "./.github/actions/boringcache" - invocation = provider.fetch("runs").fetch("steps").fetch(0) + invocation = provider.fetch("runs").fetch("steps").find { |step| step["id"] == "provider" } supplied = document.fetch("with", {}) declared = provider.fetch("inputs") unknown = supplied.keys - declared.keys @@ -506,7 +519,7 @@ def self.write_contract_views(directory, root: ROOT) # defaults and each caller's phase, adapter, and strict failure settings. provider_path = File.join(target, ".github", "actions", "boringcache", "action.yml") provider = YAML.safe_load(File.read(provider_path), aliases: true) - raise Error, "The provider wrapper must contain one product call" unless provider.dig("runs", "steps").length == 1 + raise Error, "The provider wrapper must contain one product call" unless provider.dig("runs", "steps").count { |step| step["uses"].to_s.start_with?("boringcache/one@") } == 1 Dir[File.join(target, ".github", "**", "*.{yml,yaml}")].each do |path| next if path == provider_path document = YAML.safe_load(File.read(path), aliases: true) diff --git a/scripts/benchmark-phase.rb b/scripts/benchmark-phase.rb new file mode 100644 index 000000000..b43724f21 --- /dev/null +++ b/scripts/benchmark-phase.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require_relative "benchmark-plan" + +module BenchmarkPhase + def self.scope(env, require_published: false) + id = env.fetch("BENCHMARK_ID") + lane = env.fetch("CACHE_LANE") + raise "Use fresh or rolling" unless %w[fresh rolling].include?(lane) + raise "Invalid benchmark ID" unless id.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) + series = env.fetch("BENCHMARK_SERIES_ID", "") + if !series.empty? + raise "Invalid series ID" unless series.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) + sample = env.fetch("BENCHMARK_SAMPLE") + raise "Invalid sample number" unless sample.match?(/\A[1-9][0-9]*\z/) + return "#{id}-series-#{series}" if lane == "rolling" + "#{id}-series-#{series}-s#{sample}-#{run_identity(env)}" + elsif require_published && env["PHASE"] == "warm" + scope = env.fetch("PUBLISHED_SCOPE", "") + raise "Legacy warm phase requires its published cache scope" unless scope.match?(/\A[a-z0-9][a-z0-9._-]+\z/) + scope + elsif lane == "rolling" + ref = env.fetch("GITHUB_REF_NAME").gsub(/[^A-Za-z0-9._-]/, "-").sub(/\A-/, "").sub(/-\z/, "") + "#{id}-rolling-#{ref.empty? ? 'main' : ref}" + else + "#{id}-run-#{run_identity(env)}" + end + end + + def self.run_identity(env) + run, attempt = env.values_at("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT") + raise "Run and attempt must be positive integers" unless [run, attempt].all? { |value| value&.match?(/\A[1-9][0-9]*\z/) } + "r#{run}-a#{attempt}" + end + + def self.policy(phase, publish_on_warm) + raise "Phase must be publish or warm" unless %w[publish warm].include?(phase) + raise "publish-on-warm must be true or false" unless %w[true false].include?(publish_on_warm) + publish = phase == "publish" || publish_on_warm == "true" + {"publish-cache" => publish, "trust-policy" => publish ? "publish" : "restore", "require-hit" => phase == "warm"} + end +end + +if $PROGRAM_NAME == __FILE__ + case ARGV.shift + when "scope" + puts BenchmarkPhase.scope(ENV, require_published: ARGV.delete("--require-published") == "--require-published") + when "policy" + BenchmarkPhase.policy(ENV.fetch("PHASE"), ENV.fetch("PUBLISH_ON_WARM", "false")).each do |name, value| + BenchmarkPlan.write_output(name, value) + end + else + abort "Use benchmark-phase.rb scope or policy" + end +end diff --git a/scripts/collect-series.rb b/scripts/collect-series.rb new file mode 100644 index 000000000..8097cb469 --- /dev/null +++ b/scripts/collect-series.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require_relative "benchmark-series" +require_relative "benchmark-evidence" + +module CollectSeries + def self.call(directory, evidence_directory:, sample:) + plan = BenchmarkSeries.load(directory) + verified = BenchmarkEvidence.verify(evidence_directory) + raise BenchmarkSeries::Error, "Collection requires a complete verified evidence export" unless verified["state"] == "complete" + run_id = verified.fetch("run_id").to_s + receipts = Dir[File.join(directory, "dispatches", "*.json")].map { |path| JSON.parse(File.read(path)) } + unless receipts.any? { |receipt| receipt["run_id"].to_s == run_id && receipt["repository"] == verified["repository"] && receipt["case_id"] == plan["case_id"] && receipt.dig("inputs", "series_id") == plan["series_id"] && receipt.dig("inputs", "sample").to_s == sample.to_s } + raise BenchmarkSeries::Error, "Evidence does not match a retained dispatch for this sample" + end + + manifest = JSON.parse(File.read(File.join(evidence_directory, "manifest.json"))) + records = {} + manifest.fetch("artifacts").each do |artifact| + next unless artifact.fetch("name").start_with?("phase-") + archive = File.join(evidence_directory, "artifacts", "#{artifact.fetch('id')}.zip") + names, error, status = Open3.capture3("unzip", "-Z1", archive) + raise BenchmarkSeries::Error, "Cannot list phase artifact: #{error.strip}" unless status.success? + names.lines.map(&:strip).grep(/\.json\z/).each do |name| + value = JSON.parse(read_record(archive, name)) + BenchmarkSeries.validate_record(plan, value) + unless value.dig("github", "run_id").to_s == run_id && value.dig("series", "sample") == sample + raise BenchmarkSeries::Error, "Phase artifact belongs to another run or sample" + end + target = File.join(directory, "runs", "#{sample}-#{value.fetch('strategy')}-#{value.fetch('phase')}.json") + existing = records[target] || (JSON.parse(File.read(target)) if File.exist?(target)) + raise BenchmarkSeries::Error, "Conflicting records for the same observation" if existing && existing != value + records[target] = value + end + end + completion_path = File.join(directory, "completions", "#{run_id}.json") + if File.exist?(completion_path) + completion = JSON.parse(File.read(completion_path)) + unless completion["sample"] == sample && completion["evidence_manifest_sha256"] == Digest::SHA256.file(File.join(evidence_directory, "manifest.json")).hexdigest + raise BenchmarkSeries::Error, "This run has a different completion record" + end + end + records.each { |path, value| BenchmarkSeries.write_json(path, value) unless File.exist?(path) } + completion ||= BenchmarkSeries.finish(directory, evidence_directory: evidence_directory, sample: sample) + BenchmarkSeries.report(directory) + {"records" => records.length, "completion" => completion} + end + + def self.read_record(archive, name) + unless name.match?(%r{\A[a-zA-Z0-9_./-]+\.json\z}) && !name.start_with?("/") && !name.split("/").include?("..") + raise BenchmarkSeries::Error, "Unsafe phase artifact filename" + end + Open3.popen3("unzip", "-p", archive, name) do |input, output, error, process| + input.close + content = output.read(8 * 1024 * 1024 + 1) + if content.bytesize > 8 * 1024 * 1024 + Process.kill("TERM", process.pid) + raise BenchmarkSeries::Error, "Phase record exceeds 8 MiB" + end + raise BenchmarkSeries::Error, "Cannot read phase artifact: #{error.read.strip}" unless process.value.success? + content + end + end +end diff --git a/scripts/nightly-canaries.rb b/scripts/nightly-canaries.rb index ff330ab10..3cd2f8025 100755 --- a/scripts/nightly-canaries.rb +++ b/scripts/nightly-canaries.rb @@ -121,7 +121,17 @@ def receipt(repository, run_id) def collect(summary:, benchmarks: BENCHMARKS, now: Time.now.utc) lines = ["## CLI canary benchmark results", "", "Dispatch and workload states are reported separately.", ""] passed = true - targets(benchmarks).group_by(&:first).each do |repository, expected| + monitored = benchmarks.map do |item| + case item.fetch("canary_location", "central") + when "central" then item + when "historical" + workflow = item.fetch("historical_fresh_workflow") { item.fetch("fresh_workflow").delete_prefix("#{item.fetch('case_id')}-") } + item.merge("source_repo" => item.fetch("historical_source_repo"), "fresh_workflow" => workflow, "fresh_inputs" => {}) + else + raise Error, "Canary location must be central or historical" + end + end + targets(monitored).group_by(&:first).each do |repository, expected| lines << "### #{repository}" begin parents = api("repos/#{repository}/actions/workflows/canary.yml/runs?branch=main&per_page=100").fetch("workflow_runs") diff --git a/suites/published.json b/suites/published.json index ed7c9b625..9fd02b29b 100644 --- a/suites/published.json +++ b/suites/published.json @@ -13,7 +13,8 @@ "historical_source_repo": "boringcache/benchmark-hugo", "case_id": "hugo", "historical_workflow": "hugo-benchmark.yml", - "historical_fresh_workflow": "hugo-fresh-benchmark.yml" + "historical_fresh_workflow": "hugo-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "hugo-go", @@ -29,7 +30,8 @@ "historical_source_repo": "boringcache/benchmark-hugo-go", "case_id": "hugo-go", "historical_workflow": "hugo-go-benchmark.yml", - "historical_fresh_workflow": "hugo-go-fresh-benchmark.yml" + "historical_fresh_workflow": "hugo-go-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "immich", @@ -47,7 +49,8 @@ "fresh_workflow": "native-fresh-benchmark.yml", "historical_source_repo": "boringcache/benchmark-immich", "case_id": "immich", - "historical_fresh_workflow": "immich-fresh-benchmark.yml" + "historical_fresh_workflow": "immich-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "mastodon-docker", @@ -71,7 +74,8 @@ "historical_fresh_workflow": "mastodon-fresh-benchmark.yml", "fresh_inputs": { "variant": "server" - } + }, + "canary_location": "historical" }, { "benchmark": "mastodon-streaming", @@ -89,7 +93,8 @@ "historical_fresh_workflow": "mastodon-fresh-benchmark.yml", "fresh_inputs": { "variant": "streaming" - } + }, + "canary_location": "historical" }, { "benchmark": "discourse-image-factory-amd64", @@ -171,7 +176,8 @@ "historical_fresh_workflow": "posthog-fresh-benchmark.yml", "fresh_inputs": { "variant": "layers" - } + }, + "canary_location": "historical" }, { "benchmark": "storybook", @@ -187,7 +193,8 @@ "historical_source_repo": "boringcache/benchmark-storybook", "case_id": "storybook", "historical_workflow": "storybook-benchmark.yml", - "historical_fresh_workflow": "storybook-fresh-benchmark.yml" + "historical_fresh_workflow": "storybook-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "otel-gradle", @@ -208,7 +215,8 @@ "historical_source_repo": "boringcache/benchmark-opentelemetry-java", "case_id": "opentelemetry-java", "historical_workflow": "opentelemetry-java-gradle-benchmark.yml", - "historical_fresh_workflow": "opentelemetry-java-gradle-fresh-benchmark.yml" + "historical_fresh_workflow": "opentelemetry-java-gradle-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "spring-ai-maven", @@ -228,7 +236,8 @@ "historical_source_repo": "boringcache/benchmark-spring-ai", "case_id": "spring-ai", "historical_workflow": "spring-ai-maven-benchmark.yml", - "historical_fresh_workflow": "spring-ai-maven-fresh-benchmark.yml" + "historical_fresh_workflow": "spring-ai-maven-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "grpc-bazel", @@ -249,7 +258,8 @@ "buildbuddy-cache" ], "historical_source_repo": "boringcache/benchmark-grpc", - "case_id": "grpc" + "case_id": "grpc", + "canary_location": "historical" }, { "benchmark": "zed-cargo", @@ -270,7 +280,8 @@ "case_id": "zed", "fresh_inputs": { "cache_layer": "combined" - } + }, + "canary_location": "historical" }, { "benchmark": "deno-cargo", @@ -290,7 +301,8 @@ "case_id": "deno", "fresh_inputs": { "cache_profile": "cargo-product" - } + }, + "canary_location": "historical" }, { "benchmark": "duckgres", @@ -306,7 +318,8 @@ "historical_source_repo": "boringcache/benchmark-duckgres", "case_id": "duckgres", "historical_workflow": "duckgres-benchmark.yml", - "historical_fresh_workflow": "duckgres-fresh-benchmark.yml" + "historical_fresh_workflow": "duckgres-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "chroma", @@ -328,7 +341,8 @@ "historical_source_repo": "boringcache/benchmark-chroma", "case_id": "chroma", "historical_workflow": "chroma-benchmark.yml", - "historical_fresh_workflow": "chroma-fresh-benchmark.yml" + "historical_fresh_workflow": "chroma-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "linkerd2-v2", @@ -349,7 +363,8 @@ "historical_source_repo": "boringcache/benchmark-linkerd2", "case_id": "linkerd2", "historical_workflow": "linkerd2-benchmark.yml", - "historical_fresh_workflow": "linkerd2-fresh-benchmark.yml" + "historical_fresh_workflow": "linkerd2-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "qdrant", @@ -365,7 +380,8 @@ "historical_source_repo": "boringcache/benchmark-qdrant", "case_id": "qdrant", "historical_workflow": "qdrant-benchmark.yml", - "historical_fresh_workflow": "qdrant-fresh-benchmark.yml" + "historical_fresh_workflow": "qdrant-fresh-benchmark.yml", + "canary_location": "historical" }, { "benchmark": "n8n", @@ -386,7 +402,8 @@ "historical_fresh_workflow": "n8n-fresh-benchmark.yml", "fresh_inputs": { "variant": "turbo" - } + }, + "canary_location": "historical" }, { "benchmark": "n8n-docker", @@ -404,7 +421,8 @@ "historical_fresh_workflow": "n8n-docker-fresh-benchmark.yml", "fresh_inputs": { "variant": "docker" - } + }, + "canary_location": "historical" }, { "benchmark": "n8n-runners", @@ -422,7 +440,8 @@ "historical_fresh_workflow": "n8n-docker-fresh-benchmark.yml", "fresh_inputs": { "variant": "runners" - } + }, + "canary_location": "historical" }, { "benchmark": "n8n-runners-distroless", @@ -440,6 +459,7 @@ "historical_fresh_workflow": "n8n-docker-fresh-benchmark.yml", "fresh_inputs": { "variant": "distroless" - } + }, + "canary_location": "historical" } ] diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 976b07803..735afe1cd 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -109,7 +109,7 @@ def test_hugo_loaded_image_disables_attestations_and_keeps_the_publication_recip assert_equal load == "true" ? "false" : "mode=max", command.fetch(command.index("--provenance") + 1) assert_equal load == "true" ? "false" : "true", command.fetch(command.index("--sbom") + 1) action = YAML.safe_load(File.read(File.join(directory, ".github/actions/hugo-docker-benchmark/action.yml"))) - build = action.dig("runs", "steps").find { |step| step["name"] == "Build with the GitHub Actions cache" }.fetch("with") + build = action.dig("runs", "steps").find { |step| step["id"] == "provider_build" }.fetch("with") assert_equal "${{ inputs.load_image != 'true' && 'mode=max' || 'false' }}", build.fetch("provenance") assert_equal "${{ inputs.load_image != 'true' }}", build.fetch("sbom") end diff --git a/test/benchmark_cases_test.rb b/test/benchmark_cases_test.rb index 0f5e16865..2b99d7be3 100644 --- a/test/benchmark_cases_test.rb +++ b/test/benchmark_cases_test.rb @@ -44,6 +44,20 @@ def test_variant_selects_the_workflow_input_and_rejects_conflicting_inputs end end + def test_installed_action_dependencies_do_not_change_definitions_or_prepared_source + with_root do |root| + action = File.join(root, ".github/actions/retain-product-evidence") + FileUtils.mkdir_p(File.join(action, "node_modules/client")) + File.write(File.join(action, "package-lock.json"), "locked dependencies") + File.write(File.join(action, "node_modules/client/index.js"), "installed code") + assert_equal [File.join(action, "package-lock.json")], BenchmarkCases.shared_action_files(root: root) + target = File.join(root, "prepared") + BenchmarkCases.copy_shared_actions(target, root: root) + assert_equal "locked dependencies", File.read(File.join(target, ".github/actions/retain-product-evidence/package-lock.json")) + refute Dir.exist?(File.join(target, ".github/actions/retain-product-evidence/node_modules")) + end + end + def test_zed_workflow_selects_only_the_requested_restore_variants workflow = YAML.safe_load_file(File.join(BenchmarkCases::ROOT, ".github/workflows/zed-zed-cargo-product.yml")) script = workflow.fetch("jobs").fetch("source").fetch("steps").find { |step| step["id"] == "source" }.fetch("run") diff --git a/test/benchmark_phase_test.rb b/test/benchmark_phase_test.rb new file mode 100644 index 000000000..52896c557 --- /dev/null +++ b/test/benchmark_phase_test.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "yaml" +require_relative "../scripts/benchmark-phase" + +class BenchmarkPhaseTest < Minitest::Test + def test_fresh_identity_is_shared_by_both_phases_and_isolated_between_samples + env = {"BENCHMARK_ID" => "posthog", "CACHE_LANE" => "fresh", "BENCHMARK_SERIES_ID" => "screening-01", + "BENCHMARK_SAMPLE" => "1", "GITHUB_RUN_ID" => "123", "GITHUB_RUN_ATTEMPT" => "2"} + cold = BenchmarkPhase.scope(env.merge("PHASE" => "publish")) + assert_equal "posthog-series-screening-01-s1-r123-a2", cold + assert_equal cold, BenchmarkPhase.scope(env.merge("PHASE" => "warm"), require_published: true) + refute_equal cold, BenchmarkPhase.scope(env.merge("BENCHMARK_SAMPLE" => "2")) + refute_equal cold, BenchmarkPhase.scope(env.merge("GITHUB_RUN_ATTEMPT" => "3")) + assert_equal "posthog-series-screening-01", BenchmarkPhase.scope(env.merge("CACHE_LANE" => "rolling")) + end + + def test_warm_consumption_never_exports_and_explicit_publication_applies_to_both_providers + cold = BenchmarkPhase.policy("publish", "false") + warm = BenchmarkPhase.policy("warm", "false") + advance = BenchmarkPhase.policy("warm", "true") + assert_equal({"publish-cache" => true, "trust-policy" => "publish", "require-hit" => false}, cold) + assert_equal({"publish-cache" => false, "trust-policy" => "restore", "require-hit" => true}, warm) + assert_equal({"publish-cache" => true, "trust-policy" => "publish", "require-hit" => true}, advance) + assert_raises(RuntimeError) { BenchmarkPhase.policy("other", "false") } + assert_raises(RuntimeError) { BenchmarkPhase.policy("warm", "") } + action = YAML.safe_load(File.read(File.expand_path("../.github/actions/docker-benchmark/action.yml", __dir__))) + steps = action.dig("runs", "steps") + actions = steps.find { |step| step["uses"].to_s.start_with?("docker/build-push-action@") }.fetch("with") + product = steps.find { |step| step["id"] == "provider" }.fetch("with") + assert_includes actions.fetch("cache-to"), "steps.policy.outputs.publish-cache == 'true'" + assert_equal "${{ steps.policy.outputs.trust-policy }}", product.fetch("trust-policy") + assert_equal "${{ steps.policy.outputs.require-hit }}", product.fetch("fail-on-cache-miss") + assert_equal "build_timing", steps.last.fetch("id") + Dir[File.expand_path("../cases/*/payload/.github/actions/*/action.yml", __dir__)].each do |path| + refute_includes File.read(path), "docker/build-push-action@", "#{path}: Docker provider lifecycle must be shared" + end + end +end diff --git a/test/benchmark_series_test.rb b/test/benchmark_series_test.rb index 5358bf8b3..62ad228a2 100644 --- a/test/benchmark_series_test.rb +++ b/test/benchmark_series_test.rb @@ -4,6 +4,7 @@ require "tmpdir" require_relative "../scripts/benchmark-series" require_relative "../scripts/benchmark-evidence" +require_relative "../scripts/collect-series" class BenchmarkSeriesTest < Minitest::Test def setup @@ -81,6 +82,54 @@ def test_reports_all_predeclared_samples_and_keeps_slow_measurements assert_equal "unreviewed", result["publication"] end + def phase_export(values) + directory = evidence + manifest = JSON.parse(File.read(File.join(directory, "manifest.json"))) + FileUtils.mkdir_p(File.join(directory, "artifacts")) + values.each_with_index do |value, index| + id = index + 1 + name = "phase-#{id}.json" + File.write(File.join(@root, name), JSON.generate(value)) + relative = "artifacts/#{id}.zip" + path = File.join(directory, relative) + _output, error, status = Open3.capture3("zip", "-q", path, name, chdir: @root) + raise error unless status.success? + manifest["artifacts"] << {"id" => id, "name" => "phase-#{id}"} + manifest["files"][relative] = {"bytes" => File.size(path), "sha256" => Digest::SHA256.file(path).hexdigest} + end + BenchmarkEvidence.retain_json(directory, "artifacts.json", {"total_count" => values.length, "artifacts" => manifest["artifacts"]}, manifest) + File.write(File.join(directory, "manifest.json"), JSON.generate(manifest)) + FileUtils.mkdir_p(File.join(@directory, "dispatches")) + BenchmarkSeries.write_json(File.join(@directory, "dispatches", "1-build.json"), + {"run_id" => 1, "repository" => BenchmarkCases::REPOSITORY, "case_id" => "example", "inputs" => {"series_id" => "screening", "sample" => "1"}}) + directory + end + + def test_collects_original_phase_artifacts_and_can_resume_without_overwriting + value = record + directory = phase_export([value, record(provider: "actions-cache"), value]) + result = CollectSeries.call(@directory, evidence_directory: directory, sample: 1) + assert_equal 2, result["records"] + assert_equal "success", result.dig("completion", "status") + assert_equal value, JSON.parse(File.read(File.join(@directory, "runs", "1-boringcache-commit.json"))) + assert_equal result, CollectSeries.call(@directory, evidence_directory: directory, sample: 1) + assert File.file?(File.join(@directory, "report.json")) + end + + def test_collection_preflights_conflicting_records_before_writing_any_observation + directory = phase_export([record, record(seconds: 99)]) + assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } + assert_empty Dir[File.join(@directory, "runs", "*.json")] + end + + def test_collection_requires_the_original_dispatch_and_matching_run + directory = phase_export([record.merge("github" => {"run_id" => "2"})]) + assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } + FileUtils.rm(File.join(@directory, "dispatches", "1-build.json")) + assert_raises(BenchmarkSeries::Error) { CollectSeries.call(@directory, evidence_directory: directory, sample: 1) } + assert_empty Dir[File.join(@directory, "runs", "*.json")] + end + def test_incomplete_series_reports_missing_samples_without_a_complete_claim add(record) result = BenchmarkSeries.report(@directory) diff --git a/test/native_case_test.rb b/test/native_case_test.rb index 46d1ddbf5..331b4b3ce 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -166,10 +166,10 @@ def test_native_docker_variants_use_the_shared_loaded_image_check assert_equal "true", recipe.dig("fresh_inputs", "load_image") action = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, "cases", id, "payload", recipe.fetch("action"), "action.yml"))) steps = action.dig("runs", "steps") - assert_operator steps.index { |step| step["id"] == "build_timing" }, :<, steps.index { |step| step["id"] == "output_verification" } + assert_operator steps.index { |step| step["id"] == "provider_build" }, :<, steps.index { |step| step["id"] == "output_verification" } steps.each_with_index do |step, index| next unless step["run"].to_s.match?(/\bdocker\s+(?:image|buildx\s+imagetools)\s+inspect\b|verify-docker-output\.rb/) - assert_operator steps.index { |entry| entry["id"] == "build_timing" }, :<, index, "#{id}: #{step['name']} must run after build timing" + assert_operator steps.index { |entry| entry["id"] == "provider_build" }, :<, index, "#{id}: #{step['name']} must run after build timing" end assert_operator steps.index { |step| step["id"] == "output_verification" }, :<, steps.index { |step| step["name"] == "Write the benchmark phase evidence" } end @@ -214,7 +214,7 @@ def test_posthog_legacy_warm_requires_and_keeps_the_explicit_published_scope end def run_posthog_scope(item, directory, step, environment, success: true) - FileUtils.cp(File.join(BenchmarkCases::ROOT, "scripts/scope-case-cache.rb"), File.join(directory, "scripts")) + %w[benchmark-plan benchmark-phase scope-case-cache].each { |name| FileUtils.cp(File.join(BenchmarkCases::ROOT, "scripts", "#{name}.rb"), File.join(directory, "scripts")) } File.write(File.join(directory, "benchmark-context.json"), JSON.generate(item)) FileUtils.cp(File.join(BenchmarkCases::ROOT, "cases/posthog/payload/.boringcache.toml"), directory) commands = File.join(directory, "commands") @@ -244,9 +244,14 @@ def test_unreviewed_recipe_input_and_unsafe_suffix_are_rejected def test_product_contract_view_resolves_the_wrapper_without_losing_adapter_or_failure_policy provider = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, ".github/actions/boringcache/action.yml"))) + steps = provider.dig("runs", "steps") + assert_equal "./.github/actions/retain-product-evidence", steps.first.fetch("uses") + assert_equal "provider", steps[1].fetch("id") + assert_equal "always()", steps.last.fetch("if") + assert_equal "${{ steps.provider.outputs.evidence-path }}", steps.last.dig("env", "EVIDENCE_PATH") step = {"uses" => "./.github/actions/boringcache", "id" => "cache", "with" => {"mode" => "cargo", "trust-policy" => "restore", "fail-on-cache-miss" => "true"}} resolved = BenchmarkCases.resolve_provider_steps(step, provider) - assert_equal provider.dig("runs", "steps", 0, "uses"), resolved["uses"] + assert_equal provider.dig("runs", "steps", 1, "uses"), resolved["uses"] assert_equal "cache", resolved["id"] assert_equal "cargo", resolved.dig("with", "mode") assert_equal "restore", resolved.dig("with", "trust-policy") @@ -257,8 +262,8 @@ def test_product_contract_view_resolves_the_wrapper_without_losing_adapter_or_fa assert_equal true, cold.dig("with", "fail-on-cache-error") conditional = BenchmarkCases.resolve_provider_steps({"uses" => "./.github/actions/boringcache", "with" => {"mode" => "docker", "fail-on-cache-miss" => "${{ inputs.phase == 'warm' }}"}}, provider) assert_equal "${{ inputs.phase == 'warm' }}", conditional.dig("with", "fail-on-cache-miss") - assert_equal true, provider.dig("runs", "steps", 0, "with", "fail-on-cache-error") - assert_equal "${{ inputs.fail-on-cache-miss == 'true' }}", provider.dig("runs", "steps", 0, "with", "fail-on-cache-miss") + assert_equal true, provider.dig("runs", "steps", 1, "with", "fail-on-cache-error") + assert_equal "${{ inputs.fail-on-cache-miss == 'true' }}", provider.dig("runs", "steps", 1, "with", "fail-on-cache-miss") assert_raises(BenchmarkCases::Error) { BenchmarkCases.resolve_provider_steps({"uses" => "./.github/actions/boringcache", "with" => {"obsolete" => "true"}}, provider) } end end diff --git a/test/nightly_canaries_test.rb b/test/nightly_canaries_test.rb index db0ed7291..dae90f310 100644 --- a/test/nightly_canaries_test.rb +++ b/test/nightly_canaries_test.rb @@ -235,4 +235,26 @@ def test_collector_skips_archived_benchmarks assert @runner.collect(summary: @summary, benchmarks: [BENCHMARK.merge("archived" => true)]) assert_empty @runner.requests end + + def test_collects_the_active_historical_canary_until_cutover + prepare_collection + item = BENCHMARK.merge("source_repo" => "boringcache/benchmarks", "case_id" => "example", + "fresh_workflow" => "native-fresh-benchmark.yml", "fresh_inputs" => {"variant" => "layers"}, + "historical_source_repo" => BENCHMARK.fetch("source_repo"), "historical_fresh_workflow" => "fresh.yml", + "canary_location" => "historical") + assert @runner.collect(summary: @summary, benchmarks: [item], now: Time.utc(2026, 9, 30, 12)) + assert_equal [[BENCHMARK.fetch("source_repo"), 21]], @runner.receipt_requests + refute @runner.requests.any? { |request| request[:path].start_with?("repos/boringcache/benchmarks/") } + + @runner.conclusion = "failure" + refute @runner.collect(summary: @summary, benchmarks: [item], now: Time.utc(2026, 9, 30, 12)) + assert_includes File.read(@summary), "**failure**" + end + + def test_canary_cutover_requires_a_known_location + assert_raises(NightlyCanaries::Error) do + @runner.collect(summary: @summary, benchmarks: [BENCHMARK.merge("canary_location" => "typo")]) + end + assert_empty @runner.requests + end end diff --git a/test/product_evidence_test.cjs b/test/product_evidence_test.cjs new file mode 100644 index 000000000..1ef7fc0b7 --- /dev/null +++ b/test/product_evidence_test.cjs @@ -0,0 +1,38 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { register, retain } = require('../.github/actions/retain-product-evidence/index.cjs'); + +test('retention uploads the unchanged final evidence after the product updates it', async () => { + const state = {}; + const outputs = {}; + const core = { saveState: (key, value) => state[key] = value, setOutput: (key, value) => outputs[key] = value, + getState: key => state[key], info: () => {} }; + register(core); + const directory = state['evidence-directory']; + try { + const source = path.join(directory, `boringcache-one-evidence-${'a'.repeat(64)}.json`); + fs.writeFileSync(outputs['registry-path'], source); + fs.writeFileSync(source, JSON.stringify({ schema_version: 'boringcache_one_evidence.v1', phases: { restore: {} } })); + const final = JSON.stringify({ schema_version: 'boringcache_one_evidence.v1', phases: { restore: {}, post: { save_status: 'saved' } } }); + fs.writeFileSync(source, final); + let uploaded = false; + await retain(core, { uploadArtifact: async (name, files, root, options) => { + uploaded = true; + assert.match(name, /^product-boringcache-one-evidence-/); + assert.equal(fs.readFileSync(files[0], 'utf8'), final); + assert.equal(root, directory); + assert.equal(options.retentionDays, 90); + } }); + assert.ok(uploaded); + fs.writeFileSync(source, '{}'); + await assert.rejects(retain(core, {}), /schema is unsupported/); + fs.unlinkSync(source); + await assert.rejects(retain(core, {}), /ENOENT/); + fs.writeFileSync(outputs['registry-path'], ''); + await assert.rejects(retain(core, {}), /did not return an evidence path/); + } finally { + fs.rmSync(directory, { recursive: true }); + } +}); From f6bcde907231d8e70c34d05e522e393db799713d Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sun, 4 Oct 2026 14:08:16 +0100 Subject: [PATCH 14/17] Declare shared lifecycle qualification series --- data/latest/series.json | 268 ++++++++++++++++++ .../series.json | 44 +++ .../series.json | 45 +++ .../shared-combined-lifecycle-01/series.json | 46 +++ .../shared-layers-lifecycle-01/series.json | 46 +++ 5 files changed, 449 insertions(+) create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/series.json create mode 100644 results/hugo/shared-lifecycle-qualification-01/series.json create mode 100644 results/posthog/shared-combined-lifecycle-01/series.json create mode 100644 results/posthog/shared-layers-lifecycle-01/series.json diff --git a/data/latest/series.json b/data/latest/series.json index 0d771ee7e..c95252d9a 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -2424,6 +2424,73 @@ ], "runs_without_archives": [] }, + { + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "plan_path": "results/hugo-go/shared-lifecycle-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "35a2dc05b7b87a66a9da5682af58e51e412316e72c350c17078ed04080c4a312", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "hugo-go", "series_id": "wrapper-qualification", @@ -2969,6 +3036,73 @@ ], "runs_without_archives": [] }, + { + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "plan_path": "results/hugo/shared-lifecycle-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "5a0c3c4b9e6cd65dc8a629f125534a1fcdd5ba9015d2f5b6aa3ba312145e53e4", + "definition_sha256": "8efc5a2368ac2012b1d1264006447fe93436d593e309aa867826b42a2d127750", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "immich", "series_id": "image-output-qualification-01", @@ -5469,6 +5603,140 @@ ], "runs_without_archives": [] }, + { + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "plan_path": "results/posthog/shared-combined-lifecycle-01/series.json", + "lane": "fresh", + "variant": "combined", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "683444f5363548d3815f9e55d6a56a023d9d15a9300582fe02bfeb322f2da2e7", + "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-01", + "plan_path": "results/posthog/shared-layers-lifecycle-01/series.json", + "lane": "fresh", + "variant": "layers", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "13c3bd7351df48d08972ba6d0c33f266205703b66a417ce6b3b96d8de674ae44", + "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [] + }, { "case_id": "qdrant", "series_id": "migration-screening-01", diff --git a/results/hugo-go/shared-lifecycle-qualification-01/series.json b/results/hugo-go/shared-lifecycle-qualification-01/series.json new file mode 100644 index 000000000..844e01700 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/series.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "created_at": "2026-10-04T13:08:11Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "variant": null, + "workflow_inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "plan_sha256": "35a2dc05b7b87a66a9da5682af58e51e412316e72c350c17078ed04080c4a312" +} diff --git a/results/hugo/shared-lifecycle-qualification-01/series.json b/results/hugo/shared-lifecycle-qualification-01/series.json new file mode 100644 index 000000000..8f671c116 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "created_at": "2026-10-04T13:08:11Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "8efc5a2368ac2012b1d1264006447fe93436d593e309aa867826b42a2d127750", + "variant": null, + "workflow_inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "5a0c3c4b9e6cd65dc8a629f125534a1fcdd5ba9015d2f5b6aa3ba312145e53e4" +} diff --git a/results/posthog/shared-combined-lifecycle-01/series.json b/results/posthog/shared-combined-lifecycle-01/series.json new file mode 100644 index 000000000..e5603f5cb --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "created_at": "2026-10-04T13:08:12Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "variant": "combined", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "683444f5363548d3815f9e55d6a56a023d9d15a9300582fe02bfeb322f2da2e7" +} diff --git a/results/posthog/shared-layers-lifecycle-01/series.json b/results/posthog/shared-layers-lifecycle-01/series.json new file mode 100644 index 000000000..a5164d73e --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-01/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-01", + "created_at": "2026-10-04T13:08:11Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "13c3bd7351df48d08972ba6d0c33f266205703b66a417ce6b3b96d8de674ae44" +} From d045da272fae61c96d06727459055f1a31eca48f Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sun, 4 Oct 2026 14:17:00 +0100 Subject: [PATCH 15/17] Correct shared phase outputs and product contract views --- .github/actions/docker-benchmark/action.yml | 28 +- data/latest/series.json | 451 +++++++++- migration/repositories.json | 68 ++ .../completions/37204582423.json | 13 + .../1-native-fresh-benchmark.yml.json | 34 + .../report.json | 823 ++++++++++++++++++ .../report.md | 20 + .../runs/1-actions-cache-cold.json | 161 ++++ .../runs/1-actions-cache-warm.json | 171 ++++ .../runs/1-boringcache-cold.json | 198 +++++ .../runs/1-boringcache-warm.json | 198 +++++ .../completions/37204584632.json | 17 + .../1-native-fresh-benchmark.yml.json | 34 + .../report.json | 76 ++ .../report.md | 22 + .../series.json | 45 + .../completions/37204588491.json | 17 + .../1-native-fresh-benchmark.yml.json | 38 + .../shared-combined-lifecycle-01/report.json | 76 ++ .../shared-combined-lifecycle-01/report.md | 22 + .../shared-combined-lifecycle-02/series.json | 46 + .../completions/37204586535.json | 17 + .../1-native-fresh-benchmark.yml.json | 38 + .../shared-layers-lifecycle-01/report.json | 76 ++ .../shared-layers-lifecycle-01/report.md | 22 + .../shared-layers-lifecycle-02/series.json | 46 + scripts/benchmark-cases.rb | 6 + scripts/benchmark-phase.rb | 2 +- test/benchmark_cases_test.rb | 13 + test/benchmark_phase_test.rb | 37 +- 30 files changed, 2769 insertions(+), 46 deletions(-) create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/completions/37204582423.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/report.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/report.md create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-cold.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-warm.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/hugo/shared-lifecycle-qualification-01/completions/37204584632.json create mode 100644 results/hugo/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/hugo/shared-lifecycle-qualification-01/report.json create mode 100644 results/hugo/shared-lifecycle-qualification-01/report.md create mode 100644 results/hugo/shared-lifecycle-qualification-02/series.json create mode 100644 results/posthog/shared-combined-lifecycle-01/completions/37204588491.json create mode 100644 results/posthog/shared-combined-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/posthog/shared-combined-lifecycle-01/report.json create mode 100644 results/posthog/shared-combined-lifecycle-01/report.md create mode 100644 results/posthog/shared-combined-lifecycle-02/series.json create mode 100644 results/posthog/shared-layers-lifecycle-01/completions/37204586535.json create mode 100644 results/posthog/shared-layers-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/posthog/shared-layers-lifecycle-01/report.json create mode 100644 results/posthog/shared-layers-lifecycle-01/report.md create mode 100644 results/posthog/shared-layers-lifecycle-02/series.json diff --git a/.github/actions/docker-benchmark/action.yml b/.github/actions/docker-benchmark/action.yml index aa6531aaa..d97cc37e9 100644 --- a/.github/actions/docker-benchmark/action.yml +++ b/.github/actions/docker-benchmark/action.yml @@ -47,10 +47,10 @@ outputs: value: ${{ steps.build_timing.outputs.build_seconds }} description: Build, image output and declared cache publication duration. cache-hit: - value: ${{ steps.provider.outputs.cache-hit }} + value: ${{ steps.publish.outputs.cache-hit || steps.restore.outputs.cache-hit }} description: Product-reported cache outcome, when measured. evidence-path: - value: ${{ steps.provider.outputs.evidence-path }} + value: ${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path }} description: Original product evidence file. runs: using: composite @@ -98,10 +98,10 @@ runs: secrets: ${{ inputs.secrets }} no-cache: ${{ inputs.no-cache == 'true' }} cache-from: type=gha,scope=${{ inputs.cache-scope }} - cache-to: ${{ steps.policy.outputs.publish-cache == 'true' && format('type=gha,scope={0},mode=max', inputs.cache-scope) || '' }} - - name: Build with BoringCache - id: provider - if: inputs.strategy == 'boringcache' + cache-to: ${{ steps.policy.outputs.publish_cache == 'true' && format('type=gha,scope={0},mode=max', inputs.cache-scope) || '' }} + - name: Build and publish with BoringCache + id: publish + if: inputs.strategy == 'boringcache' && steps.policy.outputs.publish_cache == 'true' uses: ./.github/actions/boringcache env: BORINGCACHE_MANAGED_BUILDKIT_IMAGE: ${{ inputs.buildkit-image }} @@ -109,8 +109,20 @@ runs: mode: docker cli-version: ${{ inputs.cli-version }} cli-platform: ${{ inputs.cli-platform }} - trust-policy: ${{ steps.policy.outputs.trust-policy }} - fail-on-cache-miss: ${{ steps.policy.outputs.require-hit }} + trust-policy: publish + fail-on-cache-miss: ${{ steps.policy.outputs.require_hit }} + - name: Build from the BoringCache seed + id: restore + if: inputs.strategy == 'boringcache' && steps.policy.outputs.publish_cache == 'false' + uses: ./.github/actions/boringcache + env: + BORINGCACHE_MANAGED_BUILDKIT_IMAGE: ${{ inputs.buildkit-image }} + with: + mode: docker + cli-version: ${{ inputs.cli-version }} + cli-platform: ${{ inputs.cli-platform }} + trust-policy: restore + fail-on-cache-miss: ${{ steps.policy.outputs.require_hit }} - name: Capture the build timing id: build_timing shell: bash diff --git a/data/latest/series.json b/data/latest/series.json index c95252d9a..ecbbac430 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -2455,41 +2455,115 @@ }, "plan_sha256": "35a2dc05b7b87a66a9da5682af58e51e412316e72c350c17078ed04080c4a312", "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", - "state": "planned", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 84, + "min": 84, + "max": 84 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 8, + "min": 8, + "max": 8 + }, + "storage": { + "median": 161784439, + "min": 161784439, + "max": 161784439 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 63, + "min": 63, + "max": 63 + }, + "storage": { + "median": 608031575, + "min": 608031575, + "max": 608031575 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": { + "median": 811503108, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 1 + } ], - "summaries": [], - "dispatches": [], - "completions": [], - "report_state": "missing", - "report_path": null, + "dispatches": [ + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-01", + "sample": "1" + }, + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:08:54Z", + "run_id": 37204582423, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204582423", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3154a932609e31cbdab09aa47b7e0671d9563ad8f438ae2a9b8b1c218c17a3d5" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/shared-lifecycle-qualification-01/report.json", "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37204582423" + ] }, { "case_id": "hugo-go", @@ -3067,6 +3141,111 @@ }, "plan_sha256": "5a0c3c4b9e6cd65dc8a629f125534a1fcdd5ba9015d2f5b6aa3ba312145e53e4", "definition_sha256": "8efc5a2368ac2012b1d1264006447fe93436d593e309aa867826b42a2d127750", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-01", + "sample": "1" + }, + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:08:56Z", + "run_id": 37204584632, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204584632", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "18e637ca4b6502539b929702e3dacedee83940a51a1a2b75c82632c4dec2ad70" + } + ], + "report_state": "current", + "report_path": "results/hugo/shared-lifecycle-qualification-01/report.json", + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [ + "37204584632" + ] + }, + { + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "plan_path": "results/hugo/shared-lifecycle-qualification-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "047e1f8caac9070ad94eb62149d671a69d3a1214a5a917c374cc2ca3e43ae8f2", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, @@ -5634,6 +5813,112 @@ }, "plan_sha256": "683444f5363548d3815f9e55d6a56a023d9d15a9300582fe02bfeb322f2da2e7", "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "shared-combined-lifecycle-01", + "sample": "1" + }, + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:09:00Z", + "run_id": 37204588491, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "sample": 1, + "run_id": "37204588491", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache posthog cold: failure", + "GitHub Actions posthog cold: failure" + ], + "evidence_manifest_sha256": "d60cbda03aeda419341ff8112992d81f32b476fbe508da86851a1f4994d2be7f" + } + ], + "report_state": "current", + "report_path": "results/posthog/shared-combined-lifecycle-01/report.json", + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [ + "37204588491" + ] + }, + { + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-02", + "plan_path": "results/posthog/shared-combined-lifecycle-02/series.json", + "lane": "fresh", + "variant": "combined", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, @@ -5701,6 +5986,112 @@ }, "plan_sha256": "13c3bd7351df48d08972ba6d0c33f266205703b66a417ce6b3b96d8de674ae44", "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "shared-layers-lifecycle-01", + "sample": "1" + }, + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:08:58Z", + "run_id": 37204586535, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-01", + "sample": 1, + "run_id": "37204586535", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog cold: failure", + "BoringCache posthog cold: failure" + ], + "evidence_manifest_sha256": "05001c070314e7cfead2f2e271c9af2fda780684c6a7ce1ff30d49b38db2e70a" + } + ], + "report_state": "current", + "report_path": "results/posthog/shared-layers-lifecycle-01/report.json", + "interpretation_path": null, + "evidence": [], + "runs_without_archives": [ + "37204586535" + ] + }, + { + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-02", + "plan_path": "results/posthog/shared-layers-lifecycle-02/series.json", + "lane": "fresh", + "variant": "layers", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, diff --git a/migration/repositories.json b/migration/repositories.json index 856451153..cca2dfa7a 100644 --- a/migration/repositories.json +++ b/migration/repositories.json @@ -48,6 +48,13 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/chroma/image-output-qualification-01/methodology-review.json" + ] } }, { @@ -287,6 +294,13 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/duckgres/image-output-qualification-01/methodology-review.json" + ] } }, { @@ -370,6 +384,13 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Original failed request and any phase records retained; corrected execution requires a separate frozen series" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/hugo/image-output-qualification-02/methodology-review.json" + ] } }, { @@ -461,6 +482,13 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/immich/image-output-qualification-01/methodology-review.json" + ] } }, { @@ -510,6 +538,13 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/linkerd2/image-output-qualification-01/methodology-review.json" + ] } }, { @@ -554,6 +589,14 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/mastodon/image-output-qualification-01/methodology-review.json", + "results/mastodon/streaming-output-qualification-01/methodology-review.json" + ] } }, { @@ -595,6 +638,15 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Selected one-sample cold/replay image materialization and completion checks passed; publication, other variants, rolling behavior, and caller cutover remain unqualified" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/n8n/distroless-output-qualification-01/methodology-review.json", + "results/n8n/image-output-qualification-01/methodology-review.json", + "results/n8n/runners-output-qualification-01/methodology-review.json" + ] } }, { @@ -735,6 +787,15 @@ ], "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", "scope": "Original failed request and any phase records retained; corrected execution requires a separate frozen series" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/posthog/image-output-combined-qualification-01/methodology-review.json", + "results/posthog/image-output-qualification-01/methodology-review.json", + "results/posthog/image-output-qualification-02/methodology-review.json" + ] } }, { @@ -777,6 +838,13 @@ ], "scope": "One cold and identical-source replay sample per provider; missing storage remains unmeasured; no publication-quality performance claim", "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03" + }, + "lifecycle_review": { + "reviewed_on": "2026-10-04", + "status": "Original output/completion evidence retained; affected Docker series are invalid for comparative claims until equal publication lifecycle is rerun", + "series_reviews": [ + "results/qdrant/migration-screening-01/methodology-review.json" + ] } }, { diff --git a/results/hugo-go/shared-lifecycle-qualification-01/completions/37204582423.json b/results/hugo-go/shared-lifecycle-qualification-01/completions/37204582423.json new file mode 100644 index 000000000..fefd8fa21 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/completions/37204582423.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204582423", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3154a932609e31cbdab09aa47b7e0671d9563ad8f438ae2a9b8b1c218c17a3d5" +} diff --git a/results/hugo-go/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/hugo-go/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..645a9034e --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-04T13:08:54Z", + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "run_id": 37204582423, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423" +} diff --git a/results/hugo-go/shared-lifecycle-qualification-01/report.json b/results/hugo-go/shared-lifecycle-qualification-01/report.json new file mode 100644 index 000000000..d02db55a8 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/report.json @@ -0,0 +1,823 @@ +{ + "schema_version": 1, + "plan_sha256": "35a2dc05b7b87a66a9da5682af58e51e412316e72c350c17078ed04080c4a312", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204582423", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3154a932609e31cbdab09aa47b7e0671d9563ad8f438ae2a9b8b1c218c17a3d5" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 84, + "min": 84, + "max": 84 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 8, + "min": 8, + "max": 8 + }, + "storage": { + "median": 161784439, + "min": 161784439, + "max": 161784439 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 63, + "min": 63, + "max": 63 + }, + "storage": { + "median": 608031575, + "min": 608031575, + "max": 608031575 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": { + "median": 811503108, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 84, + "workflow_seconds": 96, + "total_seconds": 84, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 84 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "go-hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1-6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172009", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 6, + "workflow_seconds": 21, + "total_seconds": 8, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 8 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 161784439, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "go-hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1-6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "complete": true, + "measured_bytes": 161784439, + "total_bytes": 161784439, + "observations": [ + { + "id": 8481577138, + "key": "go-hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1-6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "ref": "refs/heads/refs/tags/shared-lifecycle-2026-10-04", + "version": "239f5768d209d0800a553200cd805e5e9f208595cb41dcd711aaab37b3c092f6", + "created_at": "2026-10-04T13:11:01.207318Z", + "last_accessed_at": "2026-10-04T13:11:38.202936Z", + "size_in_bytes": 161784439 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172011", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 58, + "workflow_seconds": 79, + "total_seconds": 63, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 63 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 608031575, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "complete": true, + "measured_bytes": 608031575, + "unmeasured_tags": [], + "total_bytes": 608031575, + "observations": [ + { + "requested_tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 608031575, + "compressed_size": 608031575, + "size": 608031575 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "go", + "resolved_tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172010", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 8, + "workflow_seconds": 26, + "total_seconds": 15, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 15 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 811503108, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "complete": true, + "measured_bytes": 811503108, + "unmeasured_tags": [], + "total_bytes": 811503108, + "observations": [ + { + "requested_tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 811503108, + "compressed_size": 811503108, + "size": 811503108 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "go", + "resolved_tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172012", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] + } + ], + "exclusions": [] +} diff --git a/results/hugo-go/shared-lifecycle-qualification-01/report.md b/results/hugo-go/shared-lifecycle-qualification-01/report.md new file mode 100644 index 000000000..2c2c9f609 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/report.md @@ -0,0 +1,20 @@ +# hugo-go: shared-lifecycle-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 84 | 84–84 | unmeasured | 0 | +| warm | actions-cache | 1 | 8 | 8–8 | 161784439 | 1 | +| cold | boringcache | 1 | 63 | 63–63 | 608031575 | 1 | +| warm | boringcache | 1 | 15 | 15–15 | 811503108 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-cold.json b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..2b5228501 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-cold.json @@ -0,0 +1,161 @@ +{ + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 84, + "workflow_seconds": 96, + "total_seconds": 84, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 84 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "go-hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1-6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172009", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] +} diff --git a/results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-warm.json b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..52d4a11b4 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-actions-cache-warm.json @@ -0,0 +1,171 @@ +{ + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 6, + "workflow_seconds": 21, + "total_seconds": 8, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 8 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 161784439, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "go-hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1-6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "complete": true, + "measured_bytes": 161784439, + "total_bytes": 161784439, + "observations": [ + { + "id": 8481577138, + "key": "go-hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1-6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "ref": "refs/heads/refs/tags/shared-lifecycle-2026-10-04", + "version": "239f5768d209d0800a553200cd805e5e9f208595cb41dcd711aaab37b3c092f6", + "created_at": "2026-10-04T13:11:01.207318Z", + "last_accessed_at": "2026-10-04T13:11:38.202936Z", + "size_in_bytes": 161784439 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172011", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] +} diff --git a/results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-cold.json b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..2619f98a4 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 58, + "workflow_seconds": 79, + "total_seconds": 63, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 63 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 608031575, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "complete": true, + "measured_bytes": 608031575, + "unmeasured_tags": [], + "total_bytes": 608031575, + "observations": [ + { + "requested_tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 608031575, + "compressed_size": 608031575, + "size": 608031575 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "go", + "resolved_tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172010", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] +} diff --git a/results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-warm.json b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..473503f3c --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "benchmark": "hugo-go", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "go", + "adapter": "gocache", + "case": { + "schema_version": 1, + "case_id": "hugo-go", + "workspace": "boringcache/benchmarks", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo-go", + "branch": "main", + "revision": "d6186956df7f400627bb2566bd907431801b191c" + }, + "verification": [ + "Pinned upstream recipe and build success" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go" + }, + "historical_workflow": "hugo-go-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 10, + "native": { + "action": ".github/actions/hugo-go-benchmark", + "benchmark_id": "hugo-go", + "rolling_inputs": { + "go_version": "1.27.0" + }, + "environment": {}, + "fresh_inputs": { + "go_version": "1.27.0" + } + }, + "cache_tags": { + "hugo-go-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 7, + "build_seconds": 8, + "workflow_seconds": 26, + "total_seconds": 15, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 15 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 811503108, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "complete": true, + "measured_bytes": 811503108, + "unmeasured_tags": [], + "total_bytes": 811503108, + "observations": [ + { + "requested_tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "tag": "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 811503108, + "compressed_size": 811503108, + "size": 811503108 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "go", + "resolved_tags": [ + "hugo-go-series-shared-lifecycle-qualification-01-s1-r37204582423-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37204582423", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172012", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37204582423-1", + "series": { + "id": "shared-lifecycle-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + ] +} diff --git a/results/hugo/shared-lifecycle-qualification-01/completions/37204584632.json b/results/hugo/shared-lifecycle-qualification-01/completions/37204584632.json new file mode 100644 index 000000000..33cb394bc --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-01/completions/37204584632.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204584632", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "18e637ca4b6502539b929702e3dacedee83940a51a1a2b75c82632c4dec2ad70" +} diff --git a/results/hugo/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json b/results/hugo/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..12790927e --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-04T13:08:56Z", + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "run_id": 37204584632, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632" +} diff --git a/results/hugo/shared-lifecycle-qualification-01/report.json b/results/hugo/shared-lifecycle-qualification-01/report.json new file mode 100644 index 000000000..a042bc654 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-01/report.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "plan_sha256": "5a0c3c4b9e6cd65dc8a629f125534a1fcdd5ba9015d2f5b6aa3ba312145e53e4", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204584632", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "18e637ca4b6502539b929702e3dacedee83940a51a1a2b75c82632c4dec2ad70" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204584632", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "18e637ca4b6502539b929702e3dacedee83940a51a1a2b75c82632c4dec2ad70" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/hugo/shared-lifecycle-qualification-01/report.md b/results/hugo/shared-lifecycle-qualification-01/report.md new file mode 100644 index 000000000..df0bdbc45 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-01/report.md @@ -0,0 +1,22 @@ +# hugo: shared-lifecycle-qualification-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: 4 declared observations missing; 0 failed. Publication requires review. + +Execution: unqualified; missing or failed job completion checks. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | + +Run 37204584632 failed completion checks: + +- Workflow concluded failure +- BoringCache hugo cold: failure +- GitHub Actions hugo cold: failure + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/hugo/shared-lifecycle-qualification-02/series.json b/results/hugo/shared-lifecycle-qualification-02/series.json new file mode 100644 index 000000000..53e93bbc3 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "created_at": "2026-10-04T13:16:48Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "variant": null, + "workflow_inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "047e1f8caac9070ad94eb62149d671a69d3a1214a5a917c374cc2ca3e43ae8f2" +} diff --git a/results/posthog/shared-combined-lifecycle-01/completions/37204588491.json b/results/posthog/shared-combined-lifecycle-01/completions/37204588491.json new file mode 100644 index 000000000..6b58f008c --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-01/completions/37204588491.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "sample": 1, + "run_id": "37204588491", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache posthog cold: failure", + "GitHub Actions posthog cold: failure" + ], + "evidence_manifest_sha256": "d60cbda03aeda419341ff8112992d81f32b476fbe508da86851a1f4994d2be7f" +} diff --git a/results/posthog/shared-combined-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/shared-combined-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..c6567f387 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "shared-combined-lifecycle-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-04T13:09:00Z", + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "run_id": 37204588491, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491" +} diff --git a/results/posthog/shared-combined-lifecycle-01/report.json b/results/posthog/shared-combined-lifecycle-01/report.json new file mode 100644 index 000000000..0682a4f9a --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-01/report.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "plan_sha256": "683444f5363548d3815f9e55d6a56a023d9d15a9300582fe02bfeb322f2da2e7", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "sample": 1, + "run_id": "37204588491", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache posthog cold: failure", + "GitHub Actions posthog cold: failure" + ], + "evidence_manifest_sha256": "d60cbda03aeda419341ff8112992d81f32b476fbe508da86851a1f4994d2be7f" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "sample": 1, + "run_id": "37204588491", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache posthog cold: failure", + "GitHub Actions posthog cold: failure" + ], + "evidence_manifest_sha256": "d60cbda03aeda419341ff8112992d81f32b476fbe508da86851a1f4994d2be7f" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/posthog/shared-combined-lifecycle-01/report.md b/results/posthog/shared-combined-lifecycle-01/report.md new file mode 100644 index 000000000..6428b1399 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-01/report.md @@ -0,0 +1,22 @@ +# posthog: shared-combined-lifecycle-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: 4 declared observations missing; 0 failed. Publication requires review. + +Execution: unqualified; missing or failed job completion checks. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | + +Run 37204588491 failed completion checks: + +- Workflow concluded failure +- BoringCache posthog cold: failure +- GitHub Actions posthog cold: failure + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/shared-combined-lifecycle-02/series.json b/results/posthog/shared-combined-lifecycle-02/series.json new file mode 100644 index 000000000..574ff8870 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-02", + "created_at": "2026-10-04T13:16:49Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "variant": "combined", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2" +} diff --git a/results/posthog/shared-layers-lifecycle-01/completions/37204586535.json b/results/posthog/shared-layers-lifecycle-01/completions/37204586535.json new file mode 100644 index 000000000..9be21180c --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-01/completions/37204586535.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-01", + "sample": 1, + "run_id": "37204586535", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog cold: failure", + "BoringCache posthog cold: failure" + ], + "evidence_manifest_sha256": "05001c070314e7cfead2f2e271c9af2fda780684c6a7ce1ff30d49b38db2e70a" +} diff --git a/results/posthog/shared-layers-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/shared-layers-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..5dc6e6b5c --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-01/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "shared-layers-lifecycle-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-04T13:08:58Z", + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "run_id": 37204586535, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535" +} diff --git a/results/posthog/shared-layers-lifecycle-01/report.json b/results/posthog/shared-layers-lifecycle-01/report.json new file mode 100644 index 000000000..956effa3f --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-01/report.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "plan_sha256": "13c3bd7351df48d08972ba6d0c33f266205703b66a417ce6b3b96d8de674ae44", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-01", + "sample": 1, + "run_id": "37204586535", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog cold: failure", + "BoringCache posthog cold: failure" + ], + "evidence_manifest_sha256": "05001c070314e7cfead2f2e271c9af2fda780684c6a7ce1ff30d49b38db2e70a" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-01", + "sample": 1, + "run_id": "37204586535", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog cold: failure", + "BoringCache posthog cold: failure" + ], + "evidence_manifest_sha256": "05001c070314e7cfead2f2e271c9af2fda780684c6a7ce1ff30d49b38db2e70a" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/posthog/shared-layers-lifecycle-01/report.md b/results/posthog/shared-layers-lifecycle-01/report.md new file mode 100644 index 000000000..b9707879a --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-01/report.md @@ -0,0 +1,22 @@ +# posthog: shared-layers-lifecycle-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: 4 declared observations missing; 0 failed. Publication requires review. + +Execution: unqualified; missing or failed job completion checks. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | + +Run 37204586535 failed completion checks: + +- Workflow concluded failure +- GitHub Actions posthog cold: failure +- BoringCache posthog cold: failure + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/shared-layers-lifecycle-02/series.json b/results/posthog/shared-layers-lifecycle-02/series.json new file mode 100644 index 000000000..b1d859ef4 --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/series.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-02", + "created_at": "2026-10-04T13:16:49Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "variant": "layers", + "workflow_inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1" +} diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index a0d0e872a..805223c5c 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -514,6 +514,12 @@ def self.write_contract_views(directory, root: ROOT) end copy_shared_actions(target, root: root) NativeCase.write_action(item, directory: target, lane: "fresh", variant: native_variant) if item.dig("execution", "native") + # A copied, unused adapter is not an invocation by this case. + docker_action = File.join(target, ".github/actions/docker-benchmark") + sources = Dir[File.join(target, ".github", "**", "*.{yml,yaml}")].reject { |path| path.start_with?(docker_action + "/") } + unless sources.any? { |path| File.read(path).include?("./.github/actions/docker-benchmark") } + FileUtils.remove_entry(docker_action) + end # Legacy product guards consume the resolved public Action invocation. # Expand the shared wrapper in these disposable views, preserving its # defaults and each caller's phase, adapter, and strict failure settings. diff --git a/scripts/benchmark-phase.rb b/scripts/benchmark-phase.rb index b43724f21..2e48586b1 100644 --- a/scripts/benchmark-phase.rb +++ b/scripts/benchmark-phase.rb @@ -37,7 +37,7 @@ def self.policy(phase, publish_on_warm) raise "Phase must be publish or warm" unless %w[publish warm].include?(phase) raise "publish-on-warm must be true or false" unless %w[true false].include?(publish_on_warm) publish = phase == "publish" || publish_on_warm == "true" - {"publish-cache" => publish, "trust-policy" => publish ? "publish" : "restore", "require-hit" => phase == "warm"} + {"publish_cache" => publish, "require_hit" => phase == "warm"} end end diff --git a/test/benchmark_cases_test.rb b/test/benchmark_cases_test.rb index 2b99d7be3..216d9541a 100644 --- a/test/benchmark_cases_test.rb +++ b/test/benchmark_cases_test.rb @@ -58,6 +58,19 @@ def test_installed_action_dependencies_do_not_change_definitions_or_prepared_sou end end + def test_contract_views_check_only_the_docker_adapter_used_by_the_case + Dir.mktmpdir("benchmark-contract-view-") do |directory| + items = %w[hugo-go hugo].map { |id| BenchmarkCases.load_case(id) } + BenchmarkCases.stub(:documents, items) { BenchmarkCases.write_contract_views(directory) } + refute File.exist?(File.join(directory, "benchmark-hugo-go/.github/actions/docker-benchmark/action.yml")) + path = File.join(directory, "benchmark-hugo/.github/actions/docker-benchmark/action.yml") + steps = YAML.safe_load_file(path).dig("runs", "steps") + calls = steps.select { |step| step["uses"].to_s.start_with?("boringcache/one@") } + assert_equal %w[publish restore], calls.map { |step| step.dig("with", "trust-policy") } + assert calls.all? { |step| step.dig("with", "mode") == "docker" && step.dig("with", "fail-on-cache-error") == true } + end + end + def test_zed_workflow_selects_only_the_requested_restore_variants workflow = YAML.safe_load_file(File.join(BenchmarkCases::ROOT, ".github/workflows/zed-zed-cargo-product.yml")) script = workflow.fetch("jobs").fetch("source").fetch("steps").find { |step| step["id"] == "source" }.fetch("run") diff --git a/test/benchmark_phase_test.rb b/test/benchmark_phase_test.rb index 52896c557..aff806cce 100644 --- a/test/benchmark_phase_test.rb +++ b/test/benchmark_phase_test.rb @@ -2,9 +2,26 @@ require "minitest/autorun" require "yaml" +require "open3" +require "tmpdir" +require "rbconfig" require_relative "../scripts/benchmark-phase" class BenchmarkPhaseTest < Minitest::Test + def test_policy_entrypoint_writes_consumable_action_outputs + Dir.mktmpdir("benchmark-policy-") do |directory| + path = File.join(directory, "outputs") + [["publish", "false"], ["warm", "false"], ["warm", "true"]].each do |phase, publish| + File.write(path, "") + output, status = Open3.capture2e({"GITHUB_OUTPUT" => path, "PHASE" => phase, "PUBLISH_ON_WARM" => publish}, + RbConfig.ruby, File.expand_path("../scripts/benchmark-phase.rb", __dir__), "policy") + assert status.success?, output + values = File.readlines(path).to_h { |line| line.strip.split("=", 2) } + assert_equal BenchmarkPhase.policy(phase, publish).transform_values(&:to_s), values + end + end + end + def test_fresh_identity_is_shared_by_both_phases_and_isolated_between_samples env = {"BENCHMARK_ID" => "posthog", "CACHE_LANE" => "fresh", "BENCHMARK_SERIES_ID" => "screening-01", "BENCHMARK_SAMPLE" => "1", "GITHUB_RUN_ID" => "123", "GITHUB_RUN_ATTEMPT" => "2"} @@ -20,18 +37,24 @@ def test_warm_consumption_never_exports_and_explicit_publication_applies_to_both cold = BenchmarkPhase.policy("publish", "false") warm = BenchmarkPhase.policy("warm", "false") advance = BenchmarkPhase.policy("warm", "true") - assert_equal({"publish-cache" => true, "trust-policy" => "publish", "require-hit" => false}, cold) - assert_equal({"publish-cache" => false, "trust-policy" => "restore", "require-hit" => true}, warm) - assert_equal({"publish-cache" => true, "trust-policy" => "publish", "require-hit" => true}, advance) + assert_equal({"publish_cache" => true, "require_hit" => false}, cold) + assert_equal({"publish_cache" => false, "require_hit" => true}, warm) + assert_equal({"publish_cache" => true, "require_hit" => true}, advance) assert_raises(RuntimeError) { BenchmarkPhase.policy("other", "false") } assert_raises(RuntimeError) { BenchmarkPhase.policy("warm", "") } action = YAML.safe_load(File.read(File.expand_path("../.github/actions/docker-benchmark/action.yml", __dir__))) steps = action.dig("runs", "steps") actions = steps.find { |step| step["uses"].to_s.start_with?("docker/build-push-action@") }.fetch("with") - product = steps.find { |step| step["id"] == "provider" }.fetch("with") - assert_includes actions.fetch("cache-to"), "steps.policy.outputs.publish-cache == 'true'" - assert_equal "${{ steps.policy.outputs.trust-policy }}", product.fetch("trust-policy") - assert_equal "${{ steps.policy.outputs.require-hit }}", product.fetch("fail-on-cache-miss") + publish = steps.find { |step| step["id"] == "publish" } + restore = steps.find { |step| step["id"] == "restore" } + assert_includes actions.fetch("cache-to"), "steps.policy.outputs.publish_cache == 'true'" + assert_equal "publish", publish.dig("with", "trust-policy") + assert_equal "restore", restore.dig("with", "trust-policy") + assert_includes publish.fetch("if"), "steps.policy.outputs.publish_cache == 'true'" + assert_includes restore.fetch("if"), "steps.policy.outputs.publish_cache == 'false'" + [publish, restore].each do |step| + assert_equal "${{ steps.policy.outputs.require_hit }}", step.dig("with", "fail-on-cache-miss") + end assert_equal "build_timing", steps.last.fetch("id") Dir[File.expand_path("../cases/*/payload/.github/actions/*/action.yml", __dir__)].each do |path| refute_includes File.read(path), "docker/build-push-action@", "#{path}: Docker provider lifecycle must be shared" From 6c544527b55d54101ea5438744ae39f0486525c8 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sun, 4 Oct 2026 14:42:12 +0100 Subject: [PATCH 16/17] Preserve shared lifecycle results and document remaining cutover gates --- README.md | 4 + data/latest/series.json | 286 ++++-- docs/migration.md | 34 + docs/process.md | 22 + migration/cache-capacity-2026-10-04.json | 23 + migration/evidence.json | 80 ++ migration/repositories.json | 18 + .../interpretation.md | 7 + .../interpretation.md | 7 + .../completions/37205110890.json | 13 + .../1-native-fresh-benchmark.yml.json | 34 + .../interpretation.md | 7 + .../report.json | 833 ++++++++++++++++++ .../report.md | 20 + .../runs/1-actions-cache-cold.json | 163 ++++ .../runs/1-actions-cache-warm.json | 163 ++++ .../runs/1-boringcache-cold.json | 208 +++++ .../runs/1-boringcache-warm.json | 208 +++++ .../interpretation.md | 7 + .../1-native-fresh-benchmark.yml.json | 38 + .../interpretation.md | 7 + .../1-native-fresh-benchmark.yml.json | 38 + 22 files changed, 2163 insertions(+), 57 deletions(-) create mode 100644 migration/cache-capacity-2026-10-04.json create mode 100644 results/hugo-go/shared-lifecycle-qualification-01/interpretation.md create mode 100644 results/hugo/shared-lifecycle-qualification-01/interpretation.md create mode 100644 results/hugo/shared-lifecycle-qualification-02/completions/37205110890.json create mode 100644 results/hugo/shared-lifecycle-qualification-02/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/hugo/shared-lifecycle-qualification-02/interpretation.md create mode 100644 results/hugo/shared-lifecycle-qualification-02/report.json create mode 100644 results/hugo/shared-lifecycle-qualification-02/report.md create mode 100644 results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-cold.json create mode 100644 results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-warm.json create mode 100644 results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-cold.json create mode 100644 results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-warm.json create mode 100644 results/posthog/shared-combined-lifecycle-01/interpretation.md create mode 100644 results/posthog/shared-combined-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json create mode 100644 results/posthog/shared-layers-lifecycle-01/interpretation.md create mode 100644 results/posthog/shared-layers-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json diff --git a/README.md b/README.md index 5c0f5c74b..fdfc3603c 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ Standard comparisons use two shared fresh and rolling workflows, which also support `workflow_call`. All cases use one [BoringCache wrapper](.github/actions/boringcache/action.yml) for the product invocation and release pin. Preparation and common tool setup use shared actions; case payloads keep their upstream recipe and output checks. +Docker cases also share [provider setup, timing, and publication policy](.github/actions/docker-benchmark/action.yml). +`bin/bench collect` imports preserved phase artifacts and generates a report after +checking the dispatch and completed jobs. Original product evidence is retained +after product cleanup. ```mermaid flowchart LR diff --git a/data/latest/series.json b/data/latest/series.json index ecbbac430..a57bbda5e 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -2559,11 +2559,23 @@ ], "report_state": "current", "report_path": "results/hugo-go/shared-lifecycle-qualification-01/report.json", - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37204582423" - ] + "interpretation_path": "results/hugo-go/shared-lifecycle-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37204582423, + "inventory_state": "complete", + "verified_files": 17, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204582423.tar", + "bundle_bytes": 335872, + "sha256": "f995093328e6357a396811765a14993f43011983d20190c2b91dd0d9518e5368", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "hugo-go", @@ -3209,11 +3221,23 @@ ], "report_state": "current", "report_path": "results/hugo/shared-lifecycle-qualification-01/report.json", - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37204584632" - ] + "interpretation_path": "results/hugo/shared-lifecycle-qualification-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37204584632, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204584632.tar", + "bundle_bytes": 147968, + "sha256": "0b21b4ea8ef41cc7fc8f14200aaa13a58e398da102d4a983f5673f64d3091544", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "hugo", @@ -3246,40 +3270,122 @@ }, "plan_sha256": "047e1f8caac9070ad94eb62149d671a69d3a1214a5a917c374cc2ca3e43ae8f2", "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", - "state": "planned", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 229, + "min": 229, + "max": 229 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 17, + "min": 17, + "max": 17 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 226, + "min": 226, + "max": 226 + }, + "storage": { + "median": 411517974, + "min": 411517974, + "max": 411517974 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 27, + "min": 27, + "max": 27 + }, + "storage": { + "median": 411517974, + "min": 411517974, + "max": 411517974 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-02", + "sample": "1" + }, + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:17:46Z", + "run_id": 37205110890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "sample": 1, + "run_id": "37205110890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6f45e249bf9659845af18a99f80367c298095a20957e661418f78bba9b15c2ac" + } + ], + "report_state": "current", + "report_path": "results/hugo/shared-lifecycle-qualification-02/report.json", + "interpretation_path": "results/hugo/shared-lifecycle-qualification-02/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37205110890, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205110890.tar", + "bundle_bytes": 517120, + "sha256": "d199516ad802b33ea4d5789998a4693c224a3c992f7e59750536e4e6358cc472", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } ], - "summaries": [], - "dispatches": [], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], "runs_without_archives": [] }, { @@ -5882,11 +5988,23 @@ ], "report_state": "current", "report_path": "results/posthog/shared-combined-lifecycle-01/report.json", - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37204588491" - ] + "interpretation_path": "results/posthog/shared-combined-lifecycle-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37204588491, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204588491.tar", + "bundle_bytes": 153088, + "sha256": "cb696ffd068cbf8d95731d03230d730d084c21c43b067ddf516d74e388ef575a", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "posthog", @@ -5919,7 +6037,7 @@ }, "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2", "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -5947,13 +6065,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "shared-combined-lifecycle-02", + "sample": "1" + }, + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:17:50Z", + "run_id": 37205114269, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37205114269" + ] }, { "case_id": "posthog", @@ -6055,11 +6194,23 @@ ], "report_state": "current", "report_path": "results/posthog/shared-layers-lifecycle-01/report.json", - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37204586535" - ] + "interpretation_path": "results/posthog/shared-layers-lifecycle-01/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37204586535, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204586535.tar", + "bundle_bytes": 153088, + "sha256": "03afa636a012405786868313088132c167e9a5e3b305c0d53761604fce3f3558", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "posthog", @@ -6092,7 +6243,7 @@ }, "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1", "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", - "state": "planned", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -6120,13 +6271,34 @@ ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "shared-layers-lifecycle-02", + "sample": "1" + }, + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:17:48Z", + "run_id": 37205112800, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + } + ], "completions": [], "report_state": "missing", "report_path": null, "interpretation_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37205112800" + ] }, { "case_id": "qdrant", diff --git a/docs/migration.md b/docs/migration.md index 09d8be0bb..cd5029ed3 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -1,5 +1,29 @@ # Consolidation acceptance +## Current cutover gates + +The central definitions and shared harness exist. Active schedules, the historical +index feed, and original repositories still require a verified cutover. Use the +same [case process](process.md) for a maintained benchmark or a selected evaluation; +adding an evaluation does not require a fork or another harness. + +| Area | Current state | Remaining acceptance | +| --- | --- | --- | +| Definitions and execution | 17 maintained cases, 47 Docker workloads, and four blocked gRPC evaluation drafts; one provider wrapper and workspace | Qualify each scheduled workload and variant, including output checks and cache source lineage | +| Docker lifecycle | One shared provider/timing action; equal publication policy; original final product evidence retained after cleanup | Corrected live qualification, followed by the remaining workload and rolling paths | +| Reporting and evidence | One canonical phase/series format, verified collection, generated catalog, preserved failures and methodology reviews | Reviewed central publication feed and complete required evidence for each retiring repository | +| Product updates | One Action pin; pin-sync and product-interface checks inspect shared execution views | Complete companion integration and released-product qualification before caller activation | +| Scheduling | Monitoring reads the existing active schedules | Qualify central source-sync/canary/release callers, then retire each replaced schedule | +| Shared GitHub capacity | Serial comparisons; capacity snapshots retained | Choose retention/capacity policy and verify seed availability for rolling comparisons | +| Fork retirement | Deferred inventory and selected candidates retained | Audit unique work, evidence, external links, and active upstream contributions before each decision | + +The [October 4 capacity snapshot](../migration/cache-capacity-2026-10-04.json) +records 10,650,140,305 bytes in 274 cache entries and the API's `max_cache_size_gb` +setting of 10. This is repository context, not a per-phase measurement or proof +of eviction. It does not establish that a particular seed survived. Capacity and +seed checks remain required before activating central schedules. + + The first cutover migrates the 18 maintained `benchmark-*` repositories: 17 benchmark cases and Docker's 47 pinned workloads. Fork evaluations are deferred in `migration/forks.json`. Original forks and benchmark repositories remain @@ -84,6 +108,16 @@ the same publication-policy limitation. They also lack the original final One evidence envelope. The new provider post hook retains that envelope after product cleanup. Corrected execution requires new frozen series and live qualification. +The corrected [Hugo Go](../results/hugo-go/shared-lifecycle-qualification-01/interpretation.md) +and [Hugo Docker](../results/hugo/shared-lifecycle-qualification-02/interpretation.md) +screens pass cold/warm output, post-step completion, and final product evidence +retention. Their one-sample records preserve BoringCache's slower warm results +and missing Actions Cache storage. The three initial Docker requests failed +before building on unsupported helper output names. All five completed requests +are retained in the [verified shared-lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04), +with 53 listed files verified after download. Corrected PostHog qualification +remains pending. + Canary monitoring now follows the active historical schedules through explicit registry metadata. Its October 4 read-only collection verified all 16 repository receipts and 17 child runs successfully. Central dispatch remains manual until diff --git a/docs/process.md b/docs/process.md index 5b7a27e92..cd56ae13e 100644 --- a/docs/process.md +++ b/docs/process.md @@ -340,3 +340,25 @@ diagnostic until their source sequence, seed lineage, and central caller have been qualified. Do not enable their schedule or promote their timing merely because the workflow completed. The series reporter keeps rolling observations but marks them invalid for comparison while seed lineage is unsupported. + +## 8. Review product updates + +The immutable `boringcache/one` pin in `.github/actions/boringcache/action.yml` is +the only maintained product Action pin. Its released default owns the ordinary +CLI version. Use the existing runtime `cli_version` input for an exact release or +canary experiment; do not add separate version constants to case actions. + +For an Action or CLI update, read the released Action metadata and public plan +contract, update the one pin when needed, then run the case, workflow, report, +and product-owned interface checks. Start new screening series for affected +adapters from a committed ref. Inspect output correctness, trust state, raw final +evidence, native errors, storage coverage, and post-step completion before +switching scheduled callers. A package download or passing schema check alone +does not qualify the new product version. + +Keep previous series, pins, and observations unchanged. New fields in product +evidence must survive in the raw artifact even before the canonical reporter +uses them. Update the shared reporter when measurement semantics change, and +retain explicit unknown or unmeasured values. Fix missing product capabilities +in the product; do not reproduce its installers, cache identity, proxy lifecycle, +or internal evidence normalization in individual benchmark cases. diff --git a/migration/cache-capacity-2026-10-04.json b/migration/cache-capacity-2026-10-04.json new file mode 100644 index 000000000..5fac40930 --- /dev/null +++ b/migration/cache-capacity-2026-10-04.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "scope": "Read-only cache-capacity snapshots during lifecycle qualification. Not per-phase occupancy or evidence that eviction did not occur.", + "observations": { + "usage": { + "collected_at": "2026-10-04T13:26:35Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/usage", + "response": { + "full_name": "boringcache/benchmarks", + "active_caches_size_in_bytes": 10650140305, + "active_caches_count": 274 + } + }, + "storage-limit": { + "collected_at": "2026-10-04T13:26:35Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/storage-limit", + "response": { + "max_cache_size_gb": 10 + } + } + } +} diff --git a/migration/evidence.json b/migration/evidence.json index bcc20b61e..c07dd52b6 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -841,6 +841,86 @@ "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", "execution_qualification": "Output and completion checks passed; unequal warm publication and missing final product evidence make the series ineligible for a comparative claim", "repository_retirement": "Original benchmark repositories remain; no fork deletion is authorized by this export" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37204582423, + "verified_files": 17, + "gaps": [], + "sha256": "f995093328e6357a396811765a14993f43011983d20190c2b91dd0d9518e5368", + "bundle_bytes": 335872, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204582423.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Selected one-sample cold/warm output and completion checks passed; publication and complete migration remain unqualified", + "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37204584632, + "verified_files": 7, + "gaps": [], + "sha256": "0b21b4ea8ef41cc7fc8f14200aaa13a58e398da102d4a983f5673f64d3091544", + "bundle_bytes": 147968, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204584632.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Unqualified; original failure before build retained", + "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37204586535, + "verified_files": 7, + "gaps": [], + "sha256": "03afa636a012405786868313088132c167e9a5e3b305c0d53761604fce3f3558", + "bundle_bytes": 153088, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204586535.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Unqualified; original failure before build retained", + "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37204588491, + "verified_files": 7, + "gaps": [], + "sha256": "cb696ffd068cbf8d95731d03230d730d084c21c43b067ddf516d74e388ef575a", + "bundle_bytes": 153088, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204588491.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Unqualified; original failure before build retained", + "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37205110890, + "verified_files": 15, + "gaps": [], + "sha256": "d199516ad802b33ea4d5789998a4693c224a3c992f7e59750536e4e6358cc472", + "bundle_bytes": 517120, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205110890.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Selected one-sample cold/warm output and completion checks passed; publication and complete migration remain unqualified", + "repository_retirement": "Original repositories remain until all migration gates pass" } ] } diff --git a/migration/repositories.json b/migration/repositories.json index cca2dfa7a..8f148a2a3 100644 --- a/migration/repositories.json +++ b/migration/repositories.json @@ -391,6 +391,15 @@ "series_reviews": [ "results/hugo/image-output-qualification-02/methodology-review.json" ] + }, + "shared_lifecycle_qualification": { + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "run_ids": [ + 37205110890 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "scope": "Selected one-sample cold/warm execution and final product-evidence retention passed; publication, rolling behavior and caller cutover remain unqualified" } }, { @@ -441,6 +450,15 @@ "scope": "Direct fresh comparison with two predeclared samples; rolling and remaining caller/schedule gates pending", "series_id": "direct-dispatch-qualification", "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-direct-dispatch-2026-10-02" + }, + "shared_lifecycle_qualification": { + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "run_ids": [ + 37204582423 + ], + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "scope": "Selected one-sample cold/warm execution and final product-evidence retention passed; publication, rolling behavior and caller cutover remain unqualified" } }, { diff --git a/results/hugo-go/shared-lifecycle-qualification-01/interpretation.md b/results/hugo-go/shared-lifecycle-qualification-01/interpretation.md new file mode 100644 index 000000000..907db67f7 --- /dev/null +++ b/results/hugo-go/shared-lifecycle-qualification-01/interpretation.md @@ -0,0 +1,7 @@ +# Hugo Go shared lifecycle qualification + +[Run 37204582423](https://github.com/boringcache/benchmarks/actions/runs/37204582423) completed the declared one-sample cold/warm comparison. Original final One evidence was uploaded after product cleanup. The preserved jobs and post-step logs pass completion checks. + +The [report](report.md) retains Actions Cache/BoringCache build-and-reuse measurements of 84/63 seconds cold and 8/15 seconds warm. BoringCache is slower in this warm observation. One observation per arm and phase does not establish a repeatable performance difference. Actions Cache cold storage is unmeasured; the available provider storage values are not complete equivalent storage accounting. + +The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and 17-file scoped inventory. This qualifies the selected execution and retention path, not publication or the remaining cases. [Repository cache capacity](../../../migration/cache-capacity-2026-10-04.json) is contextual and does not prove absence of eviction. diff --git a/results/hugo/shared-lifecycle-qualification-01/interpretation.md b/results/hugo/shared-lifecycle-qualification-01/interpretation.md new file mode 100644 index 000000000..f33f2f731 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-01/interpretation.md @@ -0,0 +1,7 @@ +# Shared lifecycle request failure + +[Run 37204584632](https://github.com/boringcache/benchmarks/actions/runs/37204584632) failed before building. The shared phase helper emitted hyphenated output names that the existing output writer rejected. No phase measurements were produced, and the original failed completion remains in the [report](report.md). + +The correction uses supported output names and tests the actual helper CLI. A [separate replacement series](../shared-lifecycle-qualification-02/interpretation.md) declares the changed execution definition; this request is not rewritten or counted as successful qualification. + +The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and seven-file scoped inventory. diff --git a/results/hugo/shared-lifecycle-qualification-02/completions/37205110890.json b/results/hugo/shared-lifecycle-qualification-02/completions/37205110890.json new file mode 100644 index 000000000..64f70c12b --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/completions/37205110890.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "sample": 1, + "run_id": "37205110890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6f45e249bf9659845af18a99f80367c298095a20957e661418f78bba9b15c2ac" +} diff --git a/results/hugo/shared-lifecycle-qualification-02/dispatches/1-native-fresh-benchmark.yml.json b/results/hugo/shared-lifecycle-qualification-02/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..baff07931 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,34 @@ +{ + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-04T13:17:46Z", + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "run_id": 37205110890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890" +} diff --git a/results/hugo/shared-lifecycle-qualification-02/interpretation.md b/results/hugo/shared-lifecycle-qualification-02/interpretation.md new file mode 100644 index 000000000..34d39fd72 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/interpretation.md @@ -0,0 +1,7 @@ +# Hugo Docker shared lifecycle qualification + +[Run 37205110890](https://github.com/boringcache/benchmarks/actions/runs/37205110890) completed all four declared phases at signed harness `d045da272fae61c96d06727459055f1a31eca48f`. Both cold arms publish; both warm arms restore without publishing. The Actions Cache warm command has no cache export, and BoringCache uses restore trust with `--read-only`. Both image outputs are loaded within timing and inspected after timing. + +The [report](report.md) retains Actions Cache/BoringCache build-and-reuse measurements of 229/226 seconds cold and 17/27 seconds warm. BoringCache is slower in this warm observation. These are single observations, not a repeatable performance claim. Actions Cache storage is unmeasured, so this series cannot establish a storage comparison. + +Original final One evidence was retained after product cleanup. The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and 15-file scoped inventory. [Repository cache capacity](../../../migration/cache-capacity-2026-10-04.json) is contextual and does not establish per-phase occupancy or absence of eviction. Other workloads and rolling publication remain unqualified. diff --git a/results/hugo/shared-lifecycle-qualification-02/report.json b/results/hugo/shared-lifecycle-qualification-02/report.json new file mode 100644 index 000000000..52073e815 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/report.json @@ -0,0 +1,833 @@ +{ + "schema_version": 1, + "plan_sha256": "047e1f8caac9070ad94eb62149d671a69d3a1214a5a917c374cc2ca3e43ae8f2", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "sample": 1, + "run_id": "37205110890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6f45e249bf9659845af18a99f80367c298095a20957e661418f78bba9b15c2ac" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 229, + "min": 229, + "max": 229 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 17, + "min": 17, + "max": 17 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 226, + "min": 226, + "max": 226 + }, + "storage": { + "median": 411517974, + "min": 411517974, + "max": 411517974 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 27, + "min": 27, + "max": 27 + }, + "storage": { + "median": 411517974, + "min": 411517974, + "max": 411517974 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 220, + "workflow_seconds": 232, + "total_seconds": 229, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 229 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172029", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 12, + "workflow_seconds": 20, + "total_seconds": 17, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 17 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172031", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 226, + "workflow_seconds": 228, + "total_seconds": 226, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 226 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + }, + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411517974, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411517974, + "unmeasured_tags": [], + "total_bytes": 411517974, + "observations": [ + { + "requested_tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "0b8c353f-8273-4d1b-869f-2cb804f1c9a8", + "manifest_root_digest": "sha256:871aa1a2aaf5a80d1f56604459a47d5f327d927693194282f062aad0d45935a4", + "kv_total_size": 0, + "compressed_size": 411517974, + "size": 411517974 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172028", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] + }, + { + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 27, + "workflow_seconds": 30, + "total_seconds": 27, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 27 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": null + }, + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411517974, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411517974, + "unmeasured_tags": [], + "total_bytes": 411517974, + "observations": [ + { + "requested_tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "0b8c353f-8273-4d1b-869f-2cb804f1c9a8", + "manifest_root_digest": "sha256:871aa1a2aaf5a80d1f56604459a47d5f327d927693194282f062aad0d45935a4", + "kv_total_size": 0, + "compressed_size": 411517974, + "size": 411517974 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172030", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] + } + ], + "exclusions": [] +} diff --git a/results/hugo/shared-lifecycle-qualification-02/report.md b/results/hugo/shared-lifecycle-qualification-02/report.md new file mode 100644 index 000000000..8e97d9194 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/report.md @@ -0,0 +1,20 @@ +# hugo: shared-lifecycle-qualification-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 229 | 229–229 | unmeasured | 0 | +| warm | actions-cache | 1 | 17 | 17–17 | unmeasured | 0 | +| cold | boringcache | 1 | 226 | 226–226 | 411517974 | 1 | +| warm | boringcache | 1 | 27 | 27–27 | 411517974 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-cold.json b/results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..b1bee00ad --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-cold.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 9, + "build_seconds": 220, + "workflow_seconds": 232, + "total_seconds": 229, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 229 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172029", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] +} diff --git a/results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-warm.json b/results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..dfa8da0e3 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/runs/1-actions-cache-warm.json @@ -0,0 +1,163 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5, + "build_seconds": 12, + "workflow_seconds": 20, + "total_seconds": 17, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 17 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172031", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] +} diff --git a/results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-cold.json b/results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..63cc423e2 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-cold.json @@ -0,0 +1,208 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 226, + "workflow_seconds": 228, + "total_seconds": 226, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 226 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + }, + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411517974, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411517974, + "unmeasured_tags": [], + "total_bytes": 411517974, + "observations": [ + { + "requested_tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "0b8c353f-8273-4d1b-869f-2cb804f1c9a8", + "manifest_root_digest": "sha256:871aa1a2aaf5a80d1f56604459a47d5f327d927693194282f062aad0d45935a4", + "kv_total_size": 0, + "compressed_size": 411517974, + "size": 411517974 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172028", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] +} diff --git a/results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-warm.json b/results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..bc5f21d39 --- /dev/null +++ b/results/hugo/shared-lifecycle-qualification-02/runs/1-boringcache-warm.json @@ -0,0 +1,208 @@ +{ + "schema_version": 1, + "benchmark": "hugo", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "hugo", + "workspace": "boringcache/benchmarks", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-hugo", + "branch": "main", + "revision": "ecc41a3b3e250f5268000ddde142d61a15959e95" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-benchmark.yml" + }, + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo" + }, + "historical_workflow": "hugo-fresh-benchmark.yml" + } + ], + "sync": "upstream-head", + "sync_minute": 9, + "native": { + "action": ".github/actions/hugo-docker-benchmark", + "benchmark_id": "hugo", + "rolling_inputs": { + "push_image": "true" + }, + "environment": {}, + "fresh_inputs": { + "load_image": "true" + } + }, + "cache_tags": { + "hugo-docker-local": "{scope}-docker" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 27, + "workflow_seconds": 30, + "total_seconds": 27, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 27 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 1, + "export_tag": null + }, + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 411517974, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "complete": true, + "measured_bytes": 411517974, + "unmeasured_tags": [], + "total_bytes": 411517974, + "observations": [ + { + "requested_tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "tag": "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "0b8c353f-8273-4d1b-869f-2cb804f1c9a8", + "manifest_root_digest": "sha256:871aa1a2aaf5a80d1f56604459a47d5f327d927693194282f062aad0d45935a4", + "kv_total_size": 0, + "compressed_size": 411517974, + "size": 411517974 + } + ] + } + }, + "source": { + "repository": "gohugoio/hugo", + "sha": "6b3ba3a7e22ae2809605b8118aeed08b244f1955" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "hugo-series-shared-lifecycle-qualification-02-s1-r37205110890-a1-docker-ubuntu-24-x86_64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205110890", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172030", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205110890-1", + "series": { + "id": "shared-lifecycle-qualification-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + ] +} diff --git a/results/posthog/shared-combined-lifecycle-01/interpretation.md b/results/posthog/shared-combined-lifecycle-01/interpretation.md new file mode 100644 index 000000000..a502c439c --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-01/interpretation.md @@ -0,0 +1,7 @@ +# Shared lifecycle request failure + +[Run 37204588491](https://github.com/boringcache/benchmarks/actions/runs/37204588491) failed before building. The shared phase helper emitted hyphenated output names that the existing output writer rejected. No phase measurements were produced, and the original failed completion remains in the [report](report.md). + +The correction uses supported output names and tests the actual helper CLI. A [separate replacement series](../shared-combined-lifecycle-02/series.json) declares the changed execution definition; this request is not rewritten or counted as successful qualification. + +The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and seven-file scoped inventory. diff --git a/results/posthog/shared-combined-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/shared-combined-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..4bc8102b5 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "combined", + "series_id": "shared-combined-lifecycle-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-04T13:17:50Z", + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "run_id": 37205114269, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269" +} diff --git a/results/posthog/shared-layers-lifecycle-01/interpretation.md b/results/posthog/shared-layers-lifecycle-01/interpretation.md new file mode 100644 index 000000000..6fd1fc80d --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-01/interpretation.md @@ -0,0 +1,7 @@ +# Shared lifecycle request failure + +[Run 37204586535](https://github.com/boringcache/benchmarks/actions/runs/37204586535) failed before building. The shared phase helper emitted hyphenated output names that the existing output writer rejected. No phase measurements were produced, and the original failed completion remains in the [report](report.md). + +The correction uses supported output names and tests the actual helper CLI. A [separate replacement series](../shared-layers-lifecycle-02/series.json) declares the changed execution definition; this request is not rewritten or counted as successful qualification. + +The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and seven-file scoped inventory. diff --git a/results/posthog/shared-layers-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json b/results/posthog/shared-layers-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json new file mode 100644 index 000000000..fa78ee3b3 --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/dispatches/1-native-fresh-benchmark.yml.json @@ -0,0 +1,38 @@ +{ + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "shared-layers-lifecycle-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": [ + "layers", + "combined" + ], + "schema_version": 1, + "requested_at": "2026-10-04T13:17:48Z", + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "run_id": 37205112800, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800" +} From 44d7e35b670ded8785a8ddde30ea27f8d74789c4 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Sun, 4 Oct 2026 17:37:30 +0100 Subject: [PATCH 17/17] Preserve verified PostHog lifecycle results --- data/latest/series.json | 256 +++-- docs/migration.md | 27 +- docs/process.md | 9 +- migration/cache-capacity-2026-10-04.json | 14 +- migration/evidence.json | 32 + migration/repositories.json | 17 +- .../completions/37205114269.json | 13 + .../interpretation.md | 11 + .../shared-combined-lifecycle-02/report.json | 947 ++++++++++++++++++ .../shared-combined-lifecycle-02/report.md | 20 + .../runs/1-actions-cache-cold.json | 191 ++++ .../runs/1-actions-cache-warm.json | 191 ++++ .../runs/1-boringcache-cold.json | 237 +++++ .../runs/1-boringcache-warm.json | 237 +++++ .../completions/37205112800.json | 13 + .../interpretation.md | 9 + .../shared-layers-lifecycle-02/report.json | 947 ++++++++++++++++++ .../shared-layers-lifecycle-02/report.md | 20 + .../runs/1-actions-cache-cold.json | 191 ++++ .../runs/1-actions-cache-warm.json | 191 ++++ .../runs/1-boringcache-cold.json | 237 +++++ .../runs/1-boringcache-warm.json | 237 +++++ 22 files changed, 3971 insertions(+), 76 deletions(-) create mode 100644 results/posthog/shared-combined-lifecycle-02/completions/37205114269.json create mode 100644 results/posthog/shared-combined-lifecycle-02/interpretation.md create mode 100644 results/posthog/shared-combined-lifecycle-02/report.json create mode 100644 results/posthog/shared-combined-lifecycle-02/report.md create mode 100644 results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-cold.json create mode 100644 results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-warm.json create mode 100644 results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-cold.json create mode 100644 results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-warm.json create mode 100644 results/posthog/shared-layers-lifecycle-02/completions/37205112800.json create mode 100644 results/posthog/shared-layers-lifecycle-02/interpretation.md create mode 100644 results/posthog/shared-layers-lifecycle-02/report.json create mode 100644 results/posthog/shared-layers-lifecycle-02/report.md create mode 100644 results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-cold.json create mode 100644 results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-warm.json create mode 100644 results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-cold.json create mode 100644 results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-warm.json diff --git a/data/latest/series.json b/data/latest/series.json index a57bbda5e..a65df8f7a 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -6037,34 +6037,70 @@ }, "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2", "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1703, + "min": 1703, + "max": 1703 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 172, + "min": 172, + "max": 172 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 842, + "min": 842, + "max": 842 + }, + "storage": { + "median": 6242034116, + "min": 6242034116, + "max": 6242034116 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 145, + "min": 145, + "max": 145 + }, + "storage": { + "median": 6242034116, + "min": 6242034116, + "max": 6242034116 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "posthog", @@ -6085,14 +6121,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37205114269" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-02", + "sample": 1, + "run_id": "37205114269", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "fb53406ccf8cfb4dabca781b2ce43e740bffb2ba0aee9c24a6af88cb3fc3f8ca" + } + ], + "report_state": "current", + "report_path": "results/posthog/shared-combined-lifecycle-02/report.json", + "interpretation_path": "results/posthog/shared-combined-lifecycle-02/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37205114269, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205114269.tar", + "bundle_bytes": 742912, + "sha256": "0775152becae67e1be4aaf4082be989879111190ddc8aeb66c820bc82a6f3d31", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "posthog", @@ -6243,34 +6305,70 @@ }, "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1", "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", - "state": "requested", + "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, + "valid_for_comparison": true, + "execution_verified": true, "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1845, + "min": 1845, + "max": 1845 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 226, + "min": 226, + "max": 226 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 953, + "min": 953, + "max": 953 + }, + "storage": { + "median": 6238377174, + "min": 6238377174, + "max": 6238377174 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 139, + "min": 139, + "max": 139 + }, + "storage": { + "median": 6238377174, + "min": 6238377174, + "max": 6238377174 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { "case_id": "posthog", @@ -6291,14 +6389,40 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "interpretation_path": null, - "evidence": [], - "runs_without_archives": [ - "37205112800" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-02", + "sample": 1, + "run_id": "37205112800", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7b265e188abeb7d75f9a8be7a820e25d864b76ff43db00e9267f8606bf3f6c18" + } + ], + "report_state": "current", + "report_path": "results/posthog/shared-layers-lifecycle-02/report.json", + "interpretation_path": "results/posthog/shared-layers-lifecycle-02/interpretation.md", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37205112800, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205112800.tar", + "bundle_bytes": 738304, + "sha256": "6206f8fad70e7ef25ca0b49fc4fde438477aa7c633f3e901d7f12c992d89c6ed", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "qdrant", diff --git a/docs/migration.md b/docs/migration.md index cd5029ed3..b57dfdab6 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -10,7 +10,7 @@ adding an evaluation does not require a fork or another harness. | Area | Current state | Remaining acceptance | | --- | --- | --- | | Definitions and execution | 17 maintained cases, 47 Docker workloads, and four blocked gRPC evaluation drafts; one provider wrapper and workspace | Qualify each scheduled workload and variant, including output checks and cache source lineage | -| Docker lifecycle | One shared provider/timing action; equal publication policy; original final product evidence retained after cleanup | Corrected live qualification, followed by the remaining workload and rolling paths | +| Docker lifecycle | One shared provider/timing action; equal publication policy; corrected Hugo and both PostHog profiles pass cold/warm execution with final product evidence retained after cleanup | Qualify the remaining workload and rolling paths | | Reporting and evidence | One canonical phase/series format, verified collection, generated catalog, preserved failures and methodology reviews | Reviewed central publication feed and complete required evidence for each retiring repository | | Product updates | One Action pin; pin-sync and product-interface checks inspect shared execution views | Complete companion integration and released-product qualification before caller activation | | Scheduling | Monitoring reads the existing active schedules | Qualify central source-sync/canary/release callers, then retire each replaced schedule | @@ -20,8 +20,9 @@ adding an evaluation does not require a fork or another harness. The [October 4 capacity snapshot](../migration/cache-capacity-2026-10-04.json) records 10,650,140,305 bytes in 274 cache entries and the API's `max_cache_size_gb` setting of 10. This is repository context, not a per-phase measurement or proof -of eviction. It does not establish that a particular seed survived. Capacity and -seed checks remain required before activating central schedules. +of eviction. A later contextual snapshot records 11,885,301,507 bytes in 289 +entries. Neither snapshot establishes that a particular seed survived. Capacity +and seed checks remain required before activating central schedules. The first cutover migrates the 18 maintained `benchmark-*` repositories: @@ -115,8 +116,15 @@ retention. Their one-sample records preserve BoringCache's slower warm results and missing Actions Cache storage. The three initial Docker requests failed before building on unsupported helper output names. All five completed requests are retained in the [verified shared-lifecycle archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04), -with 53 listed files verified after download. Corrected PostHog qualification -remains pending. +with 53 listed files verified after download. Corrected PostHog +[layers](../results/posthog/shared-layers-lifecycle-02/interpretation.md) and +[combined](../results/posthog/shared-combined-lifecycle-02/interpretation.md) +screens also pass both providers' cold/warm output and completion checks. Their +original final product evidence is retained after cleanup. Actions Cache storage +is unmeasured. The combined record's selected Docker-tag bytes do not establish +coverage of tool-cache storage, and identical-source replay does not isolate +tool-cache benefit. Durable publication verification for these two exports is +recorded separately in the evidence inventory. Canary monitoring now follows the active historical schedules through explicit registry metadata. Its October 4 read-only collection verified all 16 repository @@ -156,7 +164,14 @@ One-sample screens do not approve publication, caller cutover, or deletion. GitHub's default CodeQL configuration now scans Ruby and Actions. Python was removed from its language list because this branch removes the maintained Python reporter; query suite, threat model, weekly schedule, and runner settings were -preserved. +preserved. The new evidence post hook adds maintained JavaScript on this branch. +After it reaches the default branch, verify that default CodeQL setup includes +`javascript-typescript` and its analysis passes. [GitHub automatically detects +new supported languages](https://docs.github.com/en/code-security/concepts/code-scanning/setup-types) +and can revert a failing new configuration. Correct missing coverage while +preserving Ruby and Actions, the existing query suite and threat model, weekly +schedule, and runner settings. +The hook's Node regression already runs in PR guardrails. Archiving does not preserve expiring Actions evidence. Missing evidence and unverified execution remain explicit in the inventory. diff --git a/docs/process.md b/docs/process.md index cd56ae13e..aedb101dc 100644 --- a/docs/process.md +++ b/docs/process.md @@ -162,7 +162,10 @@ parent seed and changed-source build. The workflow's `phases` declares the latte as `["cold", "commit"]`; it must never be labelled an identical-source warm replay. Rolling series contain changed-source commit observations and retain the parent cache seed. Identities include case and series. Cases share a workspace but must not share -cold cache identity accidentally. +cold cache identity accidentally. A fresh tag or scope establishes a logical +cold seed; it does not establish an empty physical provider store. Keep existing +workspace/repository storage context and do not claim zero prior stored content +from namespace isolation alone. GitHub cache quota remains repository-wide. Control concurrency and record occupancy and eviction. Key prefixes do not create separate quotas. Keep fixture @@ -238,6 +241,10 @@ reported KV or archive size; the GitHub probe uses the selected cache key's reported archive size. Keep those sources explicit. Do not infer equivalent compression, cross-tag deduplication, or physical storage efficiency from these counts alone. +Check coverage against the selected cache profile. For a combined Docker and +tool-cache profile, Docker-tag bytes alone do not establish total selected-profile +storage. State the measured subset in the interpretation and retain any missing +coverage; do not infer additional product tags in the harness. GitHub storage reporting receives the read-only token only in the reporting step. The selected exact-key rows are retained with their cache IDs, refs, versions, timestamps, and byte fields. A missing row or byte field remains unmeasured; diff --git a/migration/cache-capacity-2026-10-04.json b/migration/cache-capacity-2026-10-04.json index 5fac40930..6cb98695f 100644 --- a/migration/cache-capacity-2026-10-04.json +++ b/migration/cache-capacity-2026-10-04.json @@ -19,5 +19,17 @@ "max_cache_size_gb": 10 } } - } + }, + "follow_up_observations": [ + { + "collected_at": "2026-10-04T14:03:33Z", + "context": "After PostHog layers completed and while the combined cold phases are running; not per-phase occupancy or evidence of eviction", + "endpoint": "repos/boringcache/benchmarks/actions/cache/usage", + "response": { + "full_name": "boringcache/benchmarks", + "active_caches_size_in_bytes": 11885301507, + "active_caches_count": 289 + } + } + ] } diff --git a/migration/evidence.json b/migration/evidence.json index c07dd52b6..3c7433e4b 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -921,6 +921,38 @@ "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", "execution_qualification": "Selected one-sample cold/warm output and completion checks passed; publication and complete migration remain unqualified", "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37205112800, + "verified_files": 15, + "gaps": [], + "sha256": "6206f8fad70e7ef25ca0b49fc4fde438477aa7c633f3e901d7f12c992d89c6ed", + "bundle_bytes": 738304, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205112800.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Selected one-sample cold/warm output and completion checks passed; publication and complete migration remain unqualified", + "repository_retirement": "Original repositories remain until all migration gates pass" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37205114269, + "verified_files": 15, + "gaps": [], + "sha256": "0775152becae67e1be4aaf4082be989879111190ddc8aeb66c820bc82a6f3d31", + "bundle_bytes": 742912, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205114269.tar", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory", + "verified_on": "2026-10-04", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "execution_qualification": "Selected one-sample cold/warm output and completion checks passed; publication and complete migration remain unqualified", + "repository_retirement": "Original repositories remain until all migration gates pass" } ] } diff --git a/migration/repositories.json b/migration/repositories.json index 8f148a2a3..23e3e10f8 100644 --- a/migration/repositories.json +++ b/migration/repositories.json @@ -348,7 +348,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "Both loaded-image cold builds failed on the attestation manifest list; no verified measurements; a separate corrected loaded-image series is requested", + "consolidated_execution": "Corrected one-sample loaded-image cold/warm comparison passed with equal publication policy and final product evidence retained; original failed and invalid comparisons remain preserved; rolling and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -772,7 +772,7 @@ ], "branch_names_inventoried": true, "historical_evidence_audit": "Website-linked run exports verified; broader history and branch audit incomplete", - "consolidated_execution": "Both warm jobs failed during scope selection; original cold timings include inspection contrary to the declared scope; methodology review rejects comparison; corrected layers and combined series are requested", + "consolidated_execution": "Corrected layers and combined one-sample cold/warm comparisons passed with equal publication policy, image inspection after timing, and final product evidence retained; original failures remain preserved; rolling, publication and caller cutover remain unqualified", "caller_cutover": "pending consolidated qualification", "retirement": "keep", "gates": [ @@ -814,6 +814,19 @@ "results/posthog/image-output-qualification-01/methodology-review.json", "results/posthog/image-output-qualification-02/methodology-review.json" ] + }, + "shared_lifecycle_qualification": { + "case_id": "posthog", + "series_ids": [ + "shared-layers-lifecycle-02", + "shared-combined-lifecycle-02" + ], + "run_ids": [ + 37205112800, + 37205114269 + ], + "scope": "Selected one-sample cold/warm output and completion checks passed; Actions Cache storage unmeasured; combined Docker-tag bytes do not establish complete tool-cache storage; publication, rolling behavior and caller cutover remain unqualified", + "evidence_release": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04" } }, { diff --git a/results/posthog/shared-combined-lifecycle-02/completions/37205114269.json b/results/posthog/shared-combined-lifecycle-02/completions/37205114269.json new file mode 100644 index 000000000..fd0f61f7a --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/completions/37205114269.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-02", + "sample": 1, + "run_id": "37205114269", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "fb53406ccf8cfb4dabca781b2ce43e740bffb2ba0aee9c24a6af88cb3fc3f8ca" +} diff --git a/results/posthog/shared-combined-lifecycle-02/interpretation.md b/results/posthog/shared-combined-lifecycle-02/interpretation.md new file mode 100644 index 000000000..edb214491 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/interpretation.md @@ -0,0 +1,11 @@ +# PostHog combined shared lifecycle qualification + +[Run 37205114269](https://github.com/boringcache/benchmarks/actions/runs/37205114269) completed all four declared phases at signed harness `d045da272fae61c96d06727459055f1a31eca48f`. Both cold arms publish; both warm arms restore without publishing. The Actions Cache warm command has no cache export, and BoringCache uses restore trust with `--read-only`. Images are loaded within timing and inspected after timing. All jobs and post steps completed successfully, and original final One evidence was retained after product cleanup. + +The [report](report.md) retains Actions Cache/BoringCache build-and-reuse measurements of 1703/842 seconds cold and 172/145 seconds warm. These include provider setup, build, cache export where declared, and image loading; they are not compile-only or complete-job durations. The Actions Cache cold log records 671.7 seconds in cache export. That work is inside the declared comparison. This one-sample screen does not establish typical performance or the cause of provider differences. + +The reviewed combined recipe enables BoringCache tool and mount caching, alongside layer reuse; the Actions Cache arm uses its declared layer-cache comparator. Both warm logs show cached build steps. Identical-source replay does not isolate tool or mount cache benefit, and it does not establish changed-source reuse or rolling seed advancement. + +Actions Cache storage is unmeasured. BoringCache reports 6,242,034,116 bytes for the selected Docker tag. The original record lists that one tag; it does not establish coverage of the combined profile's tool-cache storage or total workspace usage. No total combined-storage or storage-efficiency claim follows from these bytes. [Repository cache capacity](../../../migration/cache-capacity-2026-10-04.json) is contextual and does not establish per-phase occupancy or absence of eviction. + +The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and 15-file scoped inventory. Earlier failed and methodologically invalid series remain unchanged. Publication, schedule cutover, and repository retirement remain unapproved. diff --git a/results/posthog/shared-combined-lifecycle-02/report.json b/results/posthog/shared-combined-lifecycle-02/report.json new file mode 100644 index 000000000..a4ed9b8ed --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/report.json @@ -0,0 +1,947 @@ +{ + "schema_version": 1, + "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-02", + "sample": 1, + "run_id": "37205114269", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "fb53406ccf8cfb4dabca781b2ce43e740bffb2ba0aee9c24a6af88cb3fc3f8ca" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1703, + "min": 1703, + "max": 1703 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 172, + "min": 172, + "max": 172 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 842, + "min": 842, + "max": 842 + }, + "storage": { + "median": 6242034116, + "min": 6242034116, + "max": 6242034116 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 145, + "min": 145, + "max": 145 + }, + "storage": { + "median": 6242034116, + "min": 6242034116, + "max": 6242034116 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 1691, + "workflow_seconds": 1736, + "total_seconds": 1703, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1703 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172079", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 14, + "build_seconds": 158, + "workflow_seconds": 204, + "total_seconds": 172, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 172 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172100", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 842, + "workflow_seconds": 877, + "total_seconds": 842, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 842 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + }, + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6242034116, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6242034116, + "unmeasured_tags": [], + "total_bytes": 6242034116, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9a0b44a0-692f-49b4-b64f-ea92ebef421c", + "manifest_root_digest": "sha256:5ad67abf4765411094eea020d5fcb355c552b87e3ba5cf50709f97432e46dc12", + "kv_total_size": 0, + "compressed_size": 6242034116, + "size": 6242034116 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172080", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 144, + "workflow_seconds": 177, + "total_seconds": 145, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 145 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6242034116, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6242034116, + "unmeasured_tags": [], + "total_bytes": 6242034116, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9a0b44a0-692f-49b4-b64f-ea92ebef421c", + "manifest_root_digest": "sha256:5ad67abf4765411094eea020d5fcb355c552b87e3ba5cf50709f97432e46dc12", + "kv_total_size": 0, + "compressed_size": 6242034116, + "size": 6242034116 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172101", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/shared-combined-lifecycle-02/report.md b/results/posthog/shared-combined-lifecycle-02/report.md new file mode 100644 index 000000000..3bc5635b2 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/report.md @@ -0,0 +1,20 @@ +# posthog: shared-combined-lifecycle-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 1703 | 1703–1703 | unmeasured | 0 | +| warm | actions-cache | 1 | 172 | 172–172 | unmeasured | 0 | +| cold | boringcache | 1 | 842 | 842–842 | 6242034116 | 1 | +| warm | boringcache | 1 | 145 | 145–145 | 6242034116 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-cold.json b/results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..6ee758b16 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-cold.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 1691, + "workflow_seconds": 1736, + "total_seconds": 1703, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1703 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172079", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] +} diff --git a/results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-warm.json b/results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..4973b8ccb --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/runs/1-actions-cache-warm.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 14, + "build_seconds": 158, + "workflow_seconds": 204, + "total_seconds": 172, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 172 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172100", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] +} diff --git a/results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-cold.json b/results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..76af3c907 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-cold.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 842, + "workflow_seconds": 877, + "total_seconds": 842, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 842 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + }, + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6242034116, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6242034116, + "unmeasured_tags": [], + "total_bytes": 6242034116, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9a0b44a0-692f-49b4-b64f-ea92ebef421c", + "manifest_root_digest": "sha256:5ad67abf4765411094eea020d5fcb355c552b87e3ba5cf50709f97432e46dc12", + "kv_total_size": 0, + "compressed_size": 6242034116, + "size": 6242034116 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172080", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] +} diff --git a/results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-warm.json b/results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..cae15d9e1 --- /dev/null +++ b/results/posthog/shared-combined-lifecycle-02/runs/1-boringcache-warm.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog-toolcache-mountcache", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "combined", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 144, + "workflow_seconds": 177, + "total_seconds": 145, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 145 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6242034116, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6242034116, + "unmeasured_tags": [], + "total_bytes": 6242034116, + "observations": [ + { + "requested_tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "tag": "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "9a0b44a0-692f-49b4-b64f-ea92ebef421c", + "manifest_root_digest": "sha256:5ad67abf4765411094eea020d5fcb355c552b87e3ba5cf50709f97432e46dc12", + "kv_total_size": 0, + "compressed_size": 6242034116, + "size": 6242034116 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-toolcache-mountcache-series-shared-combined-lifecycle-02-s1-r37205114269-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205114269", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172101", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205114269-1", + "series": { + "id": "shared-combined-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + ] +} diff --git a/results/posthog/shared-layers-lifecycle-02/completions/37205112800.json b/results/posthog/shared-layers-lifecycle-02/completions/37205112800.json new file mode 100644 index 000000000..14037fa3b --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/completions/37205112800.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-02", + "sample": 1, + "run_id": "37205112800", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7b265e188abeb7d75f9a8be7a820e25d864b76ff43db00e9267f8606bf3f6c18" +} diff --git a/results/posthog/shared-layers-lifecycle-02/interpretation.md b/results/posthog/shared-layers-lifecycle-02/interpretation.md new file mode 100644 index 000000000..83c9117da --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/interpretation.md @@ -0,0 +1,9 @@ +# PostHog layers shared lifecycle qualification + +[Run 37205112800](https://github.com/boringcache/benchmarks/actions/runs/37205112800) completed all four declared phases at signed harness `d045da272fae61c96d06727459055f1a31eca48f`. Both cold arms publish; both warm arms restore without publishing. The Actions Cache warm build has no cache export, and BoringCache uses restore trust with `--read-only`. Images are loaded within timing and inspected after timing. All jobs and post steps completed successfully, and original final One evidence was retained after product cleanup. + +The [report](report.md) retains Actions Cache/BoringCache build-and-reuse measurements of 1845/953 seconds cold and 226/139 seconds warm. These include provider setup, build, cache export where declared, and image loading; they are not compile-only or complete-job durations. The Actions Cache cold log records 893.2 seconds in cache export. That work is inside the declared comparison. This one-sample screen does not establish typical performance or the cause of provider differences. + +Both warm logs show cached build steps, demonstrating identical-source layer reuse. The screen does not establish changed-source reuse or rolling seed advancement. Actions Cache storage is unmeasured. BoringCache reports 6,238,377,174 bytes for the selected Docker tag; this is not physical workspace usage or a storage-efficiency comparison. [Repository cache capacity](../../../migration/cache-capacity-2026-10-04.json) is contextual and does not establish per-phase occupancy or absence of eviction. + +The [archive](https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04) was downloaded again and verified against its SHA-256 and 15-file scoped inventory. Earlier failed and methodologically invalid series remain unchanged. Publication, schedule cutover, and repository retirement remain unapproved. diff --git a/results/posthog/shared-layers-lifecycle-02/report.json b/results/posthog/shared-layers-lifecycle-02/report.json new file mode 100644 index 000000000..9b38000e3 --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/report.json @@ -0,0 +1,947 @@ +{ + "schema_version": 1, + "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "shared-layers-lifecycle-02", + "sample": 1, + "run_id": "37205112800", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7b265e188abeb7d75f9a8be7a820e25d864b76ff43db00e9267f8606bf3f6c18" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 1845, + "min": 1845, + "max": 1845 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 226, + "min": 226, + "max": 226 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 953, + "min": 953, + "max": 953 + }, + "storage": { + "median": 6238377174, + "min": 6238377174, + "max": 6238377174 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 139, + "min": 139, + "max": 139 + }, + "storage": { + "median": 6238377174, + "min": 6238377174, + "max": 6238377174 + }, + "storage_measured_count": 1 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 1835, + "workflow_seconds": 1880, + "total_seconds": 1845, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1845 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172034", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 214, + "workflow_seconds": 262, + "total_seconds": 226, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 226 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172073", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 953, + "workflow_seconds": 986, + "total_seconds": 953, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 953 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + }, + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6238377174, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6238377174, + "unmeasured_tags": [], + "total_bytes": 6238377174, + "observations": [ + { + "requested_tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "f6036eb2-9fed-48e3-98ff-0f40d050ffaa", + "manifest_root_digest": "sha256:28fb1734c8ad1590a8c631b3425271a66ad55de68316f052bb35f5372aa3a486", + "kv_total_size": 0, + "compressed_size": 6238377174, + "size": 6238377174 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172035", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] + }, + { + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 139, + "workflow_seconds": 174, + "total_seconds": 139, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 139 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6238377174, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6238377174, + "unmeasured_tags": [], + "total_bytes": 6238377174, + "observations": [ + { + "requested_tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "f6036eb2-9fed-48e3-98ff-0f40d050ffaa", + "manifest_root_digest": "sha256:28fb1734c8ad1590a8c631b3425271a66ad55de68316f052bb35f5372aa3a486", + "kv_total_size": 0, + "compressed_size": 6238377174, + "size": 6238377174 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172072", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] + } + ], + "exclusions": [] +} diff --git a/results/posthog/shared-layers-lifecycle-02/report.md b/results/posthog/shared-layers-lifecycle-02/report.md new file mode 100644 index 000000000..def247534 --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/report.md @@ -0,0 +1,20 @@ +# posthog: shared-layers-lifecycle-02 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately + +Status: all declared observations collected; 0 failed. Publication requires review. + +Execution: preserved job completion and post-step logs verified. Timings alone do not qualify the series. + +Queue time, dependency setup outside the declared scope, and job duration are context. They are excluded from the comparison. No observations were excluded. + +| Phase | Provider | Successful observations | Median build_and_reuse_seconds | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| cold | actions-cache | 1 | 1845 | 1845–1845 | unmeasured | 0 | +| warm | actions-cache | 1 | 226 | 226–226 | unmeasured | 0 | +| cold | boringcache | 1 | 953 | 953–953 | 6238377174 | 1 | +| warm | boringcache | 1 | 139 | 139–139 | 6238377174 | 1 | + +Each run record retains its source, runner environment, verification, provider storage source, and evidence links. Missing storage is unmeasured; it is not zero. Original Actions URLs remain subject to retention; durable evidence publication must be verified before publication review. diff --git a/results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-cold.json b/results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..73f97a70a --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-cold.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 10, + "build_seconds": 1835, + "workflow_seconds": 1880, + "total_seconds": 1845, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 1845 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172034", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] +} diff --git a/results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-warm.json b/results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..9cf494d9b --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/runs/1-actions-cache-warm.json @@ -0,0 +1,191 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 214, + "workflow_seconds": 262, + "total_seconds": 226, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 226 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "key": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1", + "complete": false, + "measured_bytes": 0, + "total_bytes": null, + "observations": [] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172073", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] +} diff --git a/results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-cold.json b/results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..100699cc3 --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-cold.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 953, + "workflow_seconds": 986, + "total_seconds": 953, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 953 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + }, + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6238377174, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6238377174, + "unmeasured_tags": [], + "total_bytes": 6238377174, + "observations": [ + { + "requested_tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "f6036eb2-9fed-48e3-98ff-0f40d050ffaa", + "manifest_root_digest": "sha256:28fb1734c8ad1590a8c631b3425271a66ad55de68316f052bb35f5372aa3a486", + "kv_total_size": 0, + "compressed_size": 6238377174, + "size": 6238377174 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "cold", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172035", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] +} diff --git a/results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-warm.json b/results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..267b37b26 --- /dev/null +++ b/results/posthog/shared-layers-lifecycle-02/runs/1-boringcache-warm.json @@ -0,0 +1,237 @@ +{ + "schema_version": 1, + "benchmark": "posthog", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": "layers", + "mode": "docker", + "adapter": "oci", + "case": { + "schema_version": 1, + "case_id": "posthog", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-posthog", + "branch": "main", + "revision": "19348dee23ef8b06302f95f179a524852518d373" + }, + "verification": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog" + }, + "historical_workflow": "posthog-posthog-fresh-benchmark.yml", + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" + }, + { + "path": ".github/workflows/posthog-posthog-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + }, + { + "path": ".github/workflows/posthog-posthog-fresh-benchmark.yml", + "lane": "diagnostic", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 21, + "cache_tags": { + "posthog-docker-local": "{scope}-docker", + "posthog-turbo-local": "{scope}-turbo" + }, + "native": { + "action": ".github/actions/posthog-docker-benchmark", + "benchmark_id": "posthog", + "fresh_inputs": { + "platform": "linux/amd64", + "load_image": "true" + }, + "rolling_inputs": { + "platform": "linux/amd64" + }, + "variants": { + "layers": {}, + "combined": { + "benchmark_id": "posthog-toolcache-mountcache", + "fresh_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "rolling_inputs": { + "docker_tool_cache": "true", + "docker_mount_cache": "true" + }, + "provider_flags": { + "actions-cache": { + "docker_tool_cache": "false", + "docker_mount_cache": "false" + } + } + } + } + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 139, + "workflow_seconds": 174, + "total_seconds": 139, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "build_and_reuse_seconds": 139 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": { + "import_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-ubuntu-24-x86_64" + ], + "planned_import_refs": 2, + "export_tag": null + }, + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker", + "workspace": "boringcache/benchmarks", + "storage_bytes": 6238377174, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "complete": true, + "measured_bytes": 6238377174, + "unmeasured_tags": [], + "total_bytes": 6238377174, + "observations": [ + { + "requested_tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "tag": "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64", + "status": "hit", + "cache_type": "cache_entry", + "cache_entry_id": "f6036eb2-9fed-48e3-98ff-0f40d050ffaa", + "manifest_root_digest": "sha256:28fb1734c8ad1590a8c631b3425271a66ad55de68316f052bb35f5372aa3a486", + "kv_total_size": 0, + "compressed_size": 6238377174, + "size": 6238377174 + } + ] + } + }, + "source": { + "repository": "PostHog/posthog", + "sha": "662f459147d2df2a44b008e4a9b6e75d8ac46e93" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "v1.33.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "docker", + "resolved_tags": [ + "posthog-series-shared-layers-lifecycle-02-s1-r37205112800-a1-docker-linux-amd64" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37205112800", + "run_attempt": "1", + "job": "warm", + "workflow": "Native comparison fresh", + "ref_name": "shared-lifecycle-correction-2026-10-04", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000172072", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37205112800-1", + "series": { + "id": "shared-layers-lifecycle-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "The freshly loaded output image exists in the local Docker image store" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + ] +}