diff --git a/.github/actions/nix-benchmark/action.yml b/.github/actions/nix-benchmark/action.yml index ed91c7855..874dafdb0 100644 --- a/.github/actions/nix-benchmark/action.yml +++ b/.github/actions/nix-benchmark/action.yml @@ -7,44 +7,43 @@ inputs: required: true phase: required: true + lane: + default: fresh cli-version: default: '' cachix-cache: default: '' cachix-token: default: '' + dependency-seed: + default: '' runs: using: composite steps: - - name: Verify the source recipe - shell: bash - run: ruby scripts/verify-upstream-recipe.rb . - - name: Install Nix - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 - with: - install_url: https://releases.nixos.org/nix/nix-2.35.2/install - enable_kvm: false - extra_nix_config: | - accept-flake-config = false - experimental-features = nix-command flakes + - uses: ./.github/actions/nix-environment - name: Prepare the dependency store shell: bash + env: + DEPENDENCY_SEED: ${{ inputs.dependency-seed }} run: | - test "$(nix --version)" = 'nix (Nix) 2.35.2' - ruby scripts/nix-benchmark.rb prepare + if [[ -n "$DEPENDENCY_SEED" ]]; then + ruby scripts/nix-benchmark.rb import-dependencies "$DEPENDENCY_SEED" + else + ruby scripts/nix-benchmark.rb prepare + fi - name: Scope the cache and start setup timing id: setup shell: bash env: BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} run: | scope="$(ruby scripts/benchmark-phase.rb scope)" ruby scripts/scope-case-cache.rb "$scope" echo "started_at=$(date +%s.%N)" >> "$GITHUB_OUTPUT" - name: Set up BoringCache id: publish - if: inputs.provider == 'boringcache' && inputs.phase == 'cold' + if: inputs.provider == 'boringcache' && inputs.phase != 'warm' uses: ./.github/actions/boringcache with: mode: nix @@ -104,17 +103,20 @@ runs: BENCHMARK_ID: ${{ inputs.case-id }} PROVIDER: ${{ inputs.provider }} PHASE: ${{ inputs.phase }} + CACHE_LANE: ${{ inputs.lane }} SETUP_SECONDS: ${{ steps.timing.outputs.seconds }} EVIDENCE_PATH: ${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path }} run: | source_repository="$(ruby -rjson -e 'puts JSON.parse(File.read("benchmark-context.json")).dig("source", "repository")')" source_sha="$(ruby -rjson -e 'puts JSON.parse(File.read("benchmark-context.json")).dig("source", "revision")')" build_seconds="$(ruby -rjson -e 'puts JSON.parse(File.read("nix-evidence/timing.json")).fetch("build_seconds")')" + args=() + [[ "$PHASE" != warm ]] || args+=(--cache-hit true) + [[ "$PHASE" != cold ]] || args+=(--cache-hit false) ruby scripts/benchmark-report.rb phase --verified-output \ - --benchmark "$BENCHMARK_ID" --strategy "$PROVIDER" --lane fresh --phase "$PHASE" --mode nix \ + --benchmark "$BENCHMARK_ID" --strategy "$PROVIDER" --lane "$CACHE_LANE" --phase "$PHASE" --mode nix \ --restore-or-setup-seconds "$SETUP_SECONDS" --build-seconds "$build_seconds" \ - --cache-hit "$([[ "$PHASE" == warm ]] && echo true || echo false)" \ - --source-repository "$source_repository" --source-sha "$source_sha" --evidence "$EVIDENCE_PATH" + --source-repository "$source_repository" --source-sha "$source_sha" --evidence "$EVIDENCE_PATH" "${args[@]}" - name: Retain Nix checks and logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f @@ -125,6 +127,6 @@ runs: - name: Retain phase measurements uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-fresh-${{ inputs.phase }} + name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-${{ inputs.lane }}-${{ inputs.phase }} path: benchmark-results/ if-no-files-found: error diff --git a/.github/actions/nix-environment/action.yml b/.github/actions/nix-environment/action.yml new file mode 100644 index 000000000..f062acb52 --- /dev/null +++ b/.github/actions/nix-environment/action.yml @@ -0,0 +1,19 @@ +name: Prepare Nix benchmark environment +description: Verifies the declared recipe and installs the pinned Nix version. +runs: + using: composite + steps: + - name: Verify the source recipe + shell: bash + run: ruby scripts/verify-upstream-recipe.rb . + - name: Install Nix + uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 + with: + install_url: https://releases.nixos.org/nix/nix-2.35.2/install + enable_kvm: false + extra_nix_config: | + accept-flake-config = false + experimental-features = nix-command flakes + - name: Verify the Nix version + shell: bash + run: test "$(nix --version)" = 'nix (Nix) 2.35.2' diff --git a/.github/actions/reapi-benchmark/action.yml b/.github/actions/reapi-benchmark/action.yml index 45ecf0b19..7af8bb666 100644 --- a/.github/actions/reapi-benchmark/action.yml +++ b/.github/actions/reapi-benchmark/action.yml @@ -9,6 +9,8 @@ inputs: required: true cli-version: required: true + lane: + default: fresh runs: using: composite steps: @@ -35,6 +37,22 @@ runs: with: name: reapi-store-${{ inputs.case-id }} path: reapi-store + - name: Restore the rolling comparator store + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + id: rolling-cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: reapi-store + key: reapi-${{ inputs.case-id }}-${{ env.BENCHMARK_ROLLING_SCOPE || env.BENCHMARK_SERIES_ID || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + reapi-${{ inputs.case-id }}-${{ env.BENCHMARK_ROLLING_SCOPE || env.BENCHMARK_SERIES_ID || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}- + - name: Record the restored comparator seed + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + shell: bash + env: + BENCHMARK_ID: ${{ inputs.case-id }} + RESTORED_CACHE_KEY: ${{ steps.rolling-cache.outputs.cache-matched-key }} + run: ruby scripts/reapi-setup.rb rolling-seed - name: Download cold output checks if: inputs.phase == 'warm' uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 @@ -48,7 +66,7 @@ runs: PHASE: ${{ inputs.phase }} PROVIDER: ${{ inputs.provider }} BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} BORINGCACHE_WORKSPACE: boringcache/benchmarks run: boringcache ci run --oidc-provider github-actions -- ruby scripts/reapi-client.rb build - name: Run bazel-remote @@ -58,7 +76,7 @@ runs: PHASE: ${{ inputs.phase }} PROVIDER: ${{ inputs.provider }} BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} run: ruby scripts/reapi-client.rb build - name: Verify outputs and record measurements shell: bash @@ -66,10 +84,20 @@ runs: PHASE: ${{ inputs.phase }} PROVIDER: ${{ inputs.provider }} BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} run: | ruby scripts/reapi-client.rb verify ruby scripts/reapi-setup.rb report + - name: Record the verified comparator source + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + shell: bash + run: ruby scripts/reapi-setup.rb publish-seed + - name: Publish the verified rolling comparator store + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: reapi-store + key: ${{ steps.rolling-cache.outputs.cache-primary-key }} - name: Retain the comparator store if: inputs.provider == 'bazel-remote' && inputs.phase == 'cold' uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f @@ -88,6 +116,6 @@ runs: - name: Retain phase measurements uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-fresh-${{ inputs.phase }} + name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-${{ inputs.lane }}-${{ inputs.phase }} path: benchmark-results/ if-no-files-found: error diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 47c62d872..c127267f5 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -1,38 +1,28 @@ name: Dispatch CLI Canary Cases on: + schedule: + - cron: '17 1 * * *' workflow_dispatch: inputs: cli_version: description: Optional exact published CLI canary; empty selects the latest type: string default: '' + dry_run: + description: Validate the dispatch without starting builds. + type: boolean + default: false permissions: contents: read + actions: write concurrency: group: benchmark-canary-dispatch cancel-in-progress: false jobs: dispatch: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - permissions: - contents: read - actions: write - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - name: Dispatch registered cases - env: - GH_TOKEN: ${{ github.token }} - CLI_VERSION: ${{ inputs.cli_version }} - run: ruby scripts/nightly-canaries.rb --repository "$GITHUB_REPOSITORY" --version "$CLI_VERSION" --output "$RUNNER_TEMP/nightly-canaries.json" --summary "$GITHUB_STEP_SUMMARY" - - name: Retain dispatch receipt - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f - with: - name: nightly-canaries - path: ${{ runner.temp }}/nightly-canaries.json - if-no-files-found: error - retention-days: 90 + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' + uses: ./.github/workflows/fresh-cadence.yml + with: + channel: canary + cli_version: ${{ inputs.cli_version || '' }} + dry_run: ${{ inputs.dry_run || false }} diff --git a/.github/workflows/fresh-cadence.yml b/.github/workflows/fresh-cadence.yml new file mode 100644 index 000000000..5fa0dd2b3 --- /dev/null +++ b/.github/workflows/fresh-cadence.yml @@ -0,0 +1,136 @@ +name: Dispatch fresh benchmark runs +on: + workflow_call: + inputs: + channel: + type: string + required: true + cli_version: + type: string + default: '' + dry_run: + type: boolean + default: false +permissions: + contents: read + actions: read +jobs: + plan: + name: Validate suite and CLI + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - name: Validate every target before requesting builds + env: + GH_TOKEN: ${{ github.token }} + CHANNEL: ${{ inputs.channel }} + CLI_VERSION: ${{ inputs.cli_version }} + run: | + ruby scripts/benchmark-cadence.rb --dry-run \ + --channel "$CHANNEL" --version "$CLI_VERSION" --ref "$GITHUB_REF_NAME" \ + --output "$RUNNER_TEMP/benchmark-dispatch.json" \ + --summary "$GITHUB_STEP_SUMMARY" + - id: matrix + run: | + echo "matrix=$(ruby scripts/cadence-dispatch.rb --plan "$RUNNER_TEMP/benchmark-dispatch.json" --matrix)" >> "$GITHUB_OUTPUT" + - name: Retain validated plan + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-plan + path: ${{ runner.temp }}/benchmark-dispatch.json + if-no-files-found: error + retention-days: 90 + - name: Retain failed preflight receipt + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-dispatch + path: ${{ runner.temp }}/benchmark-dispatch.json + if-no-files-found: error + retention-days: 90 + dispatch: + name: '${{ matrix.label }} / ${{ inputs.channel }} fresh' + needs: plan + if: '!inputs.dry_run' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + actions: write + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: benchmark-plan + path: ${{ runner.temp }}/plan + - name: Request this benchmark + env: + GH_TOKEN: ${{ github.token }} + TARGET_INDEX: ${{ matrix.index }} + run: | + ruby scripts/cadence-dispatch.rb \ + --plan "$RUNNER_TEMP/plan/benchmark-dispatch.json" --index "$TARGET_INDEX" \ + --output "$RUNNER_TEMP/target-$TARGET_INDEX.json" + - name: Retain this request + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-target-${{ matrix.index }} + path: ${{ runner.temp }}/target-${{ matrix.index }}.json + if-no-files-found: error + retention-days: 90 + receipt: + name: Collect requested runs + needs: [plan, dispatch] + if: always() && needs.plan.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: benchmark-plan + path: ${{ runner.temp }}/plan + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + if: '!inputs.dry_run' + with: + pattern: benchmark-target-* + path: ${{ runner.temp }}/receipts + - name: Combine requests without losing failed or missing targets + if: always() + env: + DRY_RUN: ${{ inputs.dry_run }} + run: | + args=() + [[ "$DRY_RUN" != true ]] || args+=(--dry-run) + ruby scripts/cadence-dispatch.rb \ + --plan "$RUNNER_TEMP/plan/benchmark-dispatch.json" \ + --receipts "$RUNNER_TEMP/receipts" \ + --output "$RUNNER_TEMP/benchmark-dispatch.json" \ + --summary "$GITHUB_STEP_SUMMARY" "${args[@]}" + - name: Retain dispatch receipt + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-dispatch + path: ${{ runner.temp }}/benchmark-dispatch.json + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/guardrails.yml b/.github/workflows/guardrails.yml index 82a82f5c6..4d8d0d2fe 100644 --- a/.github/workflows/guardrails.yml +++ b/.github/workflows/guardrails.yml @@ -16,6 +16,8 @@ jobs: run: bundle exec ruby bin/bench check - name: Check publication registry run: bundle exec ruby scripts/check-registry-alignment.rb + - name: Check scheduled cases + run: bundle exec ruby scripts/benchmark-cadence.rb --check - name: Check execution contracts run: bundle exec ruby scripts/check-workflow-guardrails.rb - name: Check report contracts diff --git a/.github/workflows/native-fresh-benchmark.yml b/.github/workflows/native-fresh-benchmark.yml index 58e76bf91..0ba1421f2 100644 --- a/.github/workflows/native-fresh-benchmark.yml +++ b/.github/workflows/native-fresh-benchmark.yml @@ -1,6 +1,6 @@ --- name: Native comparison fresh -run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample +run-name: "${{ inputs.case_id }} ${{ inputs.variant }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample }}" permissions: contents: read diff --git a/.github/workflows/native-rolling-benchmark.yml b/.github/workflows/native-rolling-benchmark.yml index 37056a8ae..07eadad28 100644 --- a/.github/workflows/native-rolling-benchmark.yml +++ b/.github/workflows/native-rolling-benchmark.yml @@ -1,7 +1,7 @@ --- name: Native comparison rolling -run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} ${{ inputs.variant }} rolling ${{ inputs.series_id + }} sample ${{ inputs.sample }}" permissions: contents: read actions: read @@ -10,6 +10,7 @@ env: BORINGCACHE_NO_GIT: 'true' BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" BENCHMARK_SAMPLE: "${{ inputs.sample }}" + BENCHMARK_ROLLING_SCOPE: "${{ inputs.cache_scope }}" jobs: publish: name: "${{ matrix.provider }} ${{ inputs.case_id }} commit" @@ -151,6 +152,9 @@ jobs: default: '' description: Reviewed native workload variant; required when the case declares variants. + cache_scope: + type: string + default: '' workflow_call: inputs: cli_version: @@ -188,7 +192,11 @@ jobs: type: string required: false default: '' + cache_scope: + type: string + default: '' concurrency: - group: benchmark-rolling-${{ inputs.case_id }}-${{ inputs.series_id || github.ref_name }} + group: benchmark-rolling-${{ inputs.case_id }}-${{ inputs.cache_scope || inputs.series_id + || github.ref_name }} cancel-in-progress: false queue: max diff --git a/.github/workflows/nightly-canaries.yml b/.github/workflows/nightly-canaries.yml index 3a71039bc..89cc14966 100644 --- a/.github/workflows/nightly-canaries.yml +++ b/.github/workflows/nightly-canaries.yml @@ -17,8 +17,16 @@ jobs: with: persist-credentials: false - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 - name: Check repository canaries - run: ruby scripts/nightly-canaries.rb --collect --summary "${GITHUB_STEP_SUMMARY}" + env: + CADENCE_ACTIVE: ${{ vars.BENCHMARK_CADENCE_ACTIVE }} + run: | + if [[ "$CADENCE_ACTIVE" == true ]]; then + bundle exec ruby scripts/benchmark-cadence.rb --collect --summary "$GITHUB_STEP_SUMMARY" + else + bundle exec ruby scripts/nightly-canaries.rb --collect --summary "$GITHUB_STEP_SUMMARY" + fi 'on': schedule: - cron: 47 * * * * diff --git a/.github/workflows/nix-fresh-benchmark.yml b/.github/workflows/nix-fresh-benchmark.yml index fc1b57967..0dacd26dc 100644 --- a/.github/workflows/nix-fresh-benchmark.yml +++ b/.github/workflows/nix-fresh-benchmark.yml @@ -1,37 +1,13 @@ +--- name: Nix comparison fresh -run-name: '${{ inputs.case_id }} Nix ${{ inputs.series_id }} sample ${{ inputs.sample }}' -on: - workflow_dispatch: - inputs: - case_id: - type: choice - options: [helix-nix, zed-nix] - required: true - provider: - description: A single provider is a correctness screen, not a comparison. - type: choice - options: [all, boringcache, cachix] - default: all - cachix_cache: - description: Cachix cache name; empty uses the repository variable. - type: string - default: '' - cli_version: - description: Exact released CLI or canary; empty uses the Action default. - type: string - default: '' - series_id: - type: string - default: '' - sample: - type: string - default: '1' +run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample + }}" permissions: contents: read actions: read env: - BENCHMARK_SERIES_ID: ${{ inputs.series_id }} - BENCHMARK_SAMPLE: ${{ inputs.sample }} + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" BENCHMARK_RUNNER_CLASS: ubuntu-24.04 BORINGCACHE_NO_GIT: 'true' jobs: @@ -39,29 +15,66 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - matrix: ${{ steps.providers.outputs.matrix }} + matrix: "${{ steps.providers.outputs.matrix }}" steps: - - id: providers - env: - PROVIDER: ${{ inputs.provider }} - CACHIX_CACHE: ${{ inputs.cachix_cache || vars.CACHIX_CACHE }} - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: | - if [[ "$PROVIDER" != boringcache ]]; then - test -n "$CACHIX_CACHE" || { echo 'Set the CACHIX_CACHE repository variable'; exit 1; } - test -n "$CACHIX_AUTH_TOKEN" || { echo 'Set the CACHIX_AUTH_TOKEN repository secret'; exit 1; } - fi - case "$PROVIDER" in - all) echo 'matrix=["boringcache","cachix"]' >> "$GITHUB_OUTPUT" ;; - boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; - cachix) echo 'matrix=["cachix"]' >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac - cold: - name: '${{ matrix.provider }} ${{ inputs.case_id }} cold' + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + CACHIX_CACHE: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + CACHIX_AUTH_TOKEN: "${{ secrets.CACHIX_AUTH_TOKEN }}" + run: | + if [[ "$PROVIDER" != boringcache ]]; then + test -n "$CACHIX_CACHE" || { echo 'Set the CACHIX_CACHE repository variable'; exit 1; } + test -n "$CACHIX_AUTH_TOKEN" || { echo 'Set the CACHIX_AUTH_TOKEN repository secret'; exit 1; } + fi + case "$PROVIDER" in + all) echo 'matrix=["boringcache","cachix"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + cachix) echo 'matrix=["cachix"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + dependencies: + name: "${{ inputs.case_id }} dependency seed" needs: providers runs-on: ubuntu-24.04 timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/nix-environment" + - name: Prepare one dependency store for every measured worker + run: | + ruby scripts/nix-benchmark.rb prepare + ruby scripts/nix-benchmark.rb export-dependencies + - name: Retain the common dependency store + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies/ + compression-level: 0 + if-no-files-found: error + retention-days: 7 + - name: Retain dependency preparation evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependency-evidence-${{ inputs.case_id }} + path: nix-evidence/ + if-no-files-found: error + retention-days: 90 + cold: + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} cold" + needs: + - providers + - dependencies + runs-on: ubuntu-24.04 + timeout-minutes: 180 permissions: contents: read actions: read @@ -69,26 +82,37 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/nix-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: cold - cli-version: ${{ inputs.cli_version }} - cachix-cache: ${{ inputs.cachix_cache || vars.CACHIX_CACHE }} - cachix-token: ${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN || '' }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - name: Download the common dependency store + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies + - uses: "./.github/actions/nix-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: cold + cli-version: "${{ inputs.cli_version }}" + cachix-cache: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + cachix-token: "${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN + || '' }}" + dependency-seed: nix-dependencies warm: - name: '${{ matrix.provider }} ${{ inputs.case_id }} warm' - needs: [providers, cold] + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} warm" + needs: + - providers + - cold + - dependencies runs-on: ubuntu-24.04 timeout-minutes: 180 permissions: @@ -98,41 +122,125 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/nix-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: warm - cli-version: ${{ inputs.cli_version }} - cachix-cache: ${{ inputs.cachix_cache || vars.CACHIX_CACHE }} - cachix-token: ${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN || '' }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - name: Download the common dependency store + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies + - uses: "./.github/actions/nix-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: warm + cli-version: "${{ inputs.cli_version }}" + cachix-cache: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + cachix-token: "${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN + || '' }}" + dependency-seed: nix-dependencies compare: if: inputs.provider == 'all' needs: warm runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - run: bundle install --jobs 4 --retry 2 - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 - with: - pattern: nix-*-cold - path: checks - - name: Compare provider outputs and dependency stores - env: - CASE_ID: ${{ inputs.case_id }} - run: | - bundle exec ruby -r ./scripts/nix-benchmark -e \ - 'id = ENV.fetch("CASE_ID"); NixBenchmark.compare(seed: "checks/nix-#{id}-boringcache-cold", current: "checks/nix-#{id}-cachix-cold")' + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: nix-*-cold + path: checks + - name: Compare provider outputs and dependency stores + env: + CASE_ID: "${{ inputs.case_id }}" + run: | + bundle exec ruby -r ./scripts/nix-benchmark -e \ + 'id = ENV.fetch("CASE_ID"); NixBenchmark.compare(seed: "checks/nix-#{id}-boringcache-cold", current: "checks/nix-#{id}-cachix-cold")' + report: + name: "${{ inputs.case_id }} fresh report" + needs: + - cold + - warm + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} fresh benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-fresh + path: benchmark-results/${{ inputs.case_id }}-boringcache-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain cachix result + if: inputs.provider == 'all' || inputs.provider == 'cachix' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-cachix-fresh + path: benchmark-results/${{ inputs.case_id }}-cachix-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-nix-fresh + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - helix-nix + - zed-nix + required: true + provider: + description: A single provider is a correctness screen, not a comparison. + type: choice + options: + - all + - boringcache + - cachix + default: all + cachix_cache: + description: Cachix cache name; empty uses the repository variable. + type: string + default: '' + cli_version: + description: Exact released CLI or canary; empty uses the Action default. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' diff --git a/.github/workflows/nix-rolling-benchmark.yml b/.github/workflows/nix-rolling-benchmark.yml new file mode 100644 index 000000000..59bce02d4 --- /dev/null +++ b/.github/workflows/nix-rolling-benchmark.yml @@ -0,0 +1,215 @@ +--- +name: Nix comparison rolling +run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample + }}" +permissions: + contents: read + actions: read +env: + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" + BENCHMARK_RUNNER_CLASS: ubuntu-24.04 + BORINGCACHE_NO_GIT: 'true' + BENCHMARK_ROLLING_SCOPE: "${{ inputs.cache_scope }}" +jobs: + providers: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: "${{ steps.providers.outputs.matrix }}" + steps: + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + CACHIX_CACHE: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + CACHIX_AUTH_TOKEN: "${{ secrets.CACHIX_AUTH_TOKEN }}" + run: | + if [[ "$PROVIDER" != boringcache ]]; then + test -n "$CACHIX_CACHE" || { echo 'Set the CACHIX_CACHE repository variable'; exit 1; } + test -n "$CACHIX_AUTH_TOKEN" || { echo 'Set the CACHIX_AUTH_TOKEN repository secret'; exit 1; } + fi + case "$PROVIDER" in + all) echo 'matrix=["boringcache","cachix"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + cachix) echo 'matrix=["cachix"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + dependencies: + name: "${{ inputs.case_id }} dependency seed" + needs: providers + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/nix-environment" + - name: Prepare one dependency store for every measured worker + run: | + ruby scripts/nix-benchmark.rb prepare + ruby scripts/nix-benchmark.rb export-dependencies + - name: Retain the common dependency store + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies/ + compression-level: 0 + if-no-files-found: error + retention-days: 7 + - name: Retain dependency preparation evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependency-evidence-${{ inputs.case_id }} + path: nix-evidence/ + if-no-files-found: error + retention-days: 90 + commit: + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} commit" + needs: + - providers + - dependencies + runs-on: ubuntu-24.04 + timeout-minutes: 180 + permissions: + contents: read + actions: read + id-token: write + strategy: + fail-fast: false + matrix: + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - name: Download the common dependency store + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies + - uses: "./.github/actions/nix-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: commit + cli-version: "${{ inputs.cli_version }}" + cachix-cache: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + cachix-token: "${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN + || '' }}" + lane: rolling + dependency-seed: nix-dependencies + compare: + if: inputs.provider == 'all' + needs: commit + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: nix-*-commit + path: checks + - name: Compare provider outputs and dependency stores + env: + CASE_ID: "${{ inputs.case_id }}" + run: | + bundle exec ruby -r ./scripts/nix-benchmark -e \ + 'id = ENV.fetch("CASE_ID"); NixBenchmark.compare(seed: "checks/nix-#{id}-boringcache-commit", current: "checks/nix-#{id}-cachix-commit")' + report: + name: "${{ inputs.case_id }} rolling report" + needs: + - commit + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} rolling benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-rolling + path: benchmark-results/${{ inputs.case_id }}-boringcache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain cachix result + if: inputs.provider == 'all' || inputs.provider == 'cachix' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-cachix-rolling + path: benchmark-results/${{ inputs.case_id }}-cachix-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-nix-rolling + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +concurrency: + group: benchmark-nix-rolling-${{ inputs.case_id }}-${{ inputs.cache_scope || inputs.series_id + || github.ref_name }} + cancel-in-progress: false + queue: max +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - helix-nix + - zed-nix + required: true + provider: + description: A single provider is a correctness screen, not a comparison. + type: choice + options: + - all + - boringcache + - cachix + default: all + cachix_cache: + description: Cachix cache name; empty uses the repository variable. + type: string + default: '' + cli_version: + description: Exact released CLI or canary; empty uses the Action default. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' + cache_scope: + type: string + default: '' diff --git a/.github/workflows/obs-rolling-benchmark.yml b/.github/workflows/obs-rolling-benchmark.yml new file mode 100644 index 000000000..cb84978e9 --- /dev/null +++ b/.github/workflows/obs-rolling-benchmark.yml @@ -0,0 +1,531 @@ +--- +name: OBS Studio comparison rolling +run-name: obs-studio ${{ inputs.cache_tool }} rolling ${{ inputs.series_id }} sample + ${{ inputs.sample }} +'on': + workflow_dispatch: + inputs: + cache_tool: + type: choice + options: + - ccache + - xcode + - all + default: all + cli_version: + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' + cache_scope: + type: string + default: '' + base_sha: + type: string + default: '' + head_sha: + type: string + default: '' + runner_label: + type: string + default: '' +permissions: + actions: read + contents: read +env: + BENCHMARK_ID: obs-studio-compiler-cache + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" + CACHE_COHORT: "${{ inputs.cache_scope || inputs.series_id || github.ref_name }}" +concurrency: + group: benchmark-obs-rolling-${{ inputs.cache_scope || inputs.series_id || github.ref_name + }} + cancel-in-progress: false + queue: max +jobs: + actions-cache-ccache: + name: GitHub Actions obs-studio ccache commit + runs-on: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' + && inputs.runner_label || 'ubuntu-26.04' }}" + timeout-minutes: 120 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + CCACHE_BASEDIR: "${{ github.workspace }}/upstream" + CCACHE_NOHASHDIR: '1' + CCACHE_COMPILERCHECK: content + CCACHE_STATSLOG: "${{ github.workspace }}/.ccache-stats.log" + BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name + == 'main' && inputs.runner_label || 'ubuntu-26.04' }}" + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source + run: "./scripts/prepare-source.sh rolling" + - name: Install the upstream Linux build shell + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> "$GITHUB_PATH" + brew install --quiet zsh + - name: Install the benchmark ccache release + run: "./scripts/install-ccache.sh" + - name: Start cache restore timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore ccache directory + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: |- + .ccache + .rolling-seed.json + key: gha-obs-ccache-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ runner.arch + }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: gha-obs-ccache-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ + runner.arch }}- + id: cache-restore + - name: Capture cache restore timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Retain the previous cache source + run: if [[ -f .rolling-seed.json ]]; then cp .rolling-seed.json benchmark-results/previous-cache-source.json; + fi + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/ccache-actions-cache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb ccache --working-directory upstream \ + 2>&1 | tee benchmark-results/ccache-actions-cache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/ccache-actions-cache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Capture and verify ccache evidence + run: | + ccache --print-log-stats --format=json > benchmark-results/ccache-actions-cache-rolling-native.json + ./scripts/verify_ccache_evidence.rb \ + --path benchmark-results/ccache-actions-cache-rolling-native.json \ + --phase commit \ + --strategy actions-cache + - name: Record the verified cache source + env: + SOURCE_SHA: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + run: 'ruby -rjson -e ''File.write(".rolling-seed.json", JSON.pretty_generate({source_sha: + ENV.fetch("SOURCE_SHA"), run_id: ENV.fetch("GITHUB_RUN_ID"), run_attempt: + ENV.fetch("GITHUB_RUN_ATTEMPT")}))''' + - name: Publish the verified ccache cache + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + key: "${{ steps.cache-restore.outputs.cache-primary-key }}" + path: |- + .ccache + .rolling-seed.json + - name: Write rolling result + env: + GITHUB_TOKEN: "${{ github.token }}" + RESTORED_CACHE_HIT: "${{ steps.cache-restore.outputs.cache-matched-key != + '' }}" + run: | + ./scripts/write_phase_result.rb \ + --surface ccache \ + --strategy actions-cache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --cache-key "${{ steps.cache-restore.outputs.cache-matched-key }}" \ + --native-evidence benchmark-results/ccache-actions-cache-rolling-native.json + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-actions-cache-ccache-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + if: inputs.cache_tool != 'xcode' && (inputs.runner_label == '' || github.ref_name + == 'main') + actions-cache-xcode: + name: GitHub Actions obs-studio xcode commit + runs-on: macos-26 + timeout-minutes: 120 + env: + XCODE_CAS_PATH: "${{ github.workspace }}/.xcode-compilation-cache" + OBS_BASELINE_DERIVED_DATA_PATH: "${{ github.workspace }}/.xcode-derived-data" + BENCHMARK_RUNNER_CLASS: macos-26 + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source + run: "./scripts/prepare-source.sh rolling" + - name: Start cache restore timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore Xcode compilation cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: |- + .xcode-compilation-cache + .rolling-seed.json + key: gha-obs-xcode-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ runner.arch + }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: gha-obs-xcode-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ runner.arch + }}- + id: cache-restore + - name: Capture cache restore timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Retain the previous cache source + run: if [[ -f .rolling-seed.json ]]; then cp .rolling-seed.json benchmark-results/previous-cache-source.json; + fi + - name: Select upstream's pinned Xcode + run: sudo xcode-select --switch "${{ steps.source.outputs.OBS_XCODE_PATH }}" + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/xcode-actions-cache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb xcode --working-directory upstream \ + 2>&1 | tee benchmark-results/xcode-actions-cache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/xcode-actions-cache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Require native Xcode build evidence + run: "./scripts/verify_xcode_build_log.rb benchmark-results/xcode-actions-cache-rolling.log" + - name: Record the verified cache source + env: + SOURCE_SHA: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + run: 'ruby -rjson -e ''File.write(".rolling-seed.json", JSON.pretty_generate({source_sha: + ENV.fetch("SOURCE_SHA"), run_id: ENV.fetch("GITHUB_RUN_ID"), run_attempt: + ENV.fetch("GITHUB_RUN_ATTEMPT")}))''' + - name: Publish the verified xcode cache + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + key: "${{ steps.cache-restore.outputs.cache-primary-key }}" + path: |- + .xcode-compilation-cache + .rolling-seed.json + - name: Write rolling result + env: + GITHUB_TOKEN: "${{ github.token }}" + RESTORED_CACHE_HIT: "${{ steps.cache-restore.outputs.cache-matched-key != + '' }}" + run: | + ./scripts/write_phase_result.rb \ + --surface xcode \ + --strategy actions-cache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --cache-key "${{ steps.cache-restore.outputs.cache-matched-key }}" + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-actions-cache-xcode-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + if: inputs.cache_tool != 'ccache' && (inputs.runner_label == '' || github.ref_name + == 'main') + boringcache-ccache: + name: BoringCache obs-studio ccache commit + runs-on: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' + && inputs.runner_label || 'ubuntu-26.04' }}" + timeout-minutes: 120 + env: + CCACHE_DIR: "${{ github.workspace }}/.boringcache-local-ccache" + CCACHE_BASEDIR: "${{ github.workspace }}/upstream" + CCACHE_NOHASHDIR: '1' + CCACHE_COMPILERCHECK: content + BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name + == 'main' && inputs.runner_label || 'ubuntu-26.04' }}" + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source and matching cache plan + run: | + ./scripts/prepare-source.sh rolling + ruby scripts/scope-case-cache.rb "obs-studio-$CACHE_COHORT" + - name: Install the upstream Linux build shell + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> "$GITHUB_PATH" + brew install --quiet zsh + - name: Install the benchmark ccache release + run: "./scripts/install-ccache.sh" + - name: Start cache setup timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore and publish BoringCache ccache + id: cache + uses: "./.github/actions/boringcache" + with: + cli-version: "${{ inputs.cli_version }}" + trust-policy: publish + mode: ccache + working-directory: "./upstream" + - name: Capture cache setup timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/ccache-boringcache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb ccache --working-directory upstream \ + 2>&1 | tee benchmark-results/ccache-boringcache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/ccache-boringcache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Capture and verify native remote evidence + run: | + ccache --print-log-stats --format=json > benchmark-results/ccache-boringcache-rolling-native.json + ./scripts/verify_ccache_evidence.rb \ + --path benchmark-results/ccache-boringcache-rolling-native.json \ + --phase commit \ + --strategy boringcache + - name: Write rolling result + run: | + ./scripts/write_phase_result.rb \ + --surface ccache \ + --strategy boringcache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --action-evidence "${{ steps.cache.outputs.evidence-path }}" \ + --native-evidence benchmark-results/ccache-boringcache-rolling-native.json + env: + RESTORED_CACHE_HIT: "${{ steps.cache.outputs.cache-hit }}" + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-boringcache-ccache-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + - name: Retain BoringCache product evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: boringcache-product-ccache-rolling + path: "${{ steps.cache.outputs.evidence-path }}" + if-no-files-found: error + retention-days: 30 + if: always() + permissions: + actions: read + contents: read + id-token: write + if: inputs.cache_tool != 'xcode' && (inputs.runner_label == '' || github.ref_name + == 'main') + boringcache-xcode: + name: BoringCache obs-studio xcode commit + if: inputs.cache_tool != 'ccache' && (inputs.runner_label == '' || github.ref_name + == 'main') + runs-on: macos-26 + timeout-minutes: 120 + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source and matching Xcode cohort + run: | + ./scripts/prepare-source.sh rolling + ruby scripts/scope-case-cache.rb "obs-studio-$CACHE_COHORT" + sudo xcode-select --switch "${{ steps.source.outputs.OBS_XCODE_PATH }}" + - name: Start cache setup timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore and publish BoringCache xcode + id: cache + uses: "./.github/actions/boringcache" + with: + cli-version: "${{ inputs.cli_version }}" + trust-policy: publish + mode: xcode + working-directory: "./upstream" + - name: Capture cache setup timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/xcode-boringcache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb xcode --working-directory upstream \ + 2>&1 | tee benchmark-results/xcode-boringcache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/xcode-boringcache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Require native Xcode build evidence + run: "./scripts/verify_xcode_build_log.rb benchmark-results/xcode-boringcache-rolling.log" + - name: Write rolling result + run: | + ./scripts/write_phase_result.rb \ + --surface xcode \ + --strategy boringcache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --action-evidence "${{ steps.cache.outputs.evidence-path }}" + env: + RESTORED_CACHE_HIT: "${{ steps.cache.outputs.cache-hit }}" + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-boringcache-xcode-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + - name: Retain BoringCache product evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: boringcache-product-xcode-rolling + path: "${{ steps.cache.outputs.evidence-path }}" + if-no-files-found: error + retention-days: 30 + if: always() + permissions: + actions: read + contents: read + id-token: write + env: + BENCHMARK_RUNNER_CLASS: macos-26 + report: + name: obs-studio rolling report + needs: + - actions-cache-ccache + - actions-cache-xcode + - boringcache-ccache + - boringcache-xcode + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-obs-studio-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: obs-studio rolling benchmark + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain actions-cache ccache result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'ccache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-actions-cache-ccache-rolling + path: benchmark-results/obs-studio-actions-cache-ccache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain actions-cache xcode result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'xcode' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-actions-cache-xcode-rolling + path: benchmark-results/obs-studio-actions-cache-xcode-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain boringcache ccache result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'ccache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-boringcache-ccache-rolling + path: benchmark-results/obs-studio-boringcache-ccache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain boringcache xcode result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'xcode' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-boringcache-xcode-rolling + path: benchmark-results/obs-studio-boringcache-xcode-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-obs-rolling + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/reapi-fresh-benchmark.yml b/.github/workflows/reapi-fresh-benchmark.yml index ac54f6100..b1b322b74 100644 --- a/.github/workflows/reapi-fresh-benchmark.yml +++ b/.github/workflows/reapi-fresh-benchmark.yml @@ -1,33 +1,13 @@ -name: Native REAPI screening -run-name: '${{ inputs.case_id }} REAPI ${{ inputs.series_id }} sample ${{ inputs.sample }}' -on: - workflow_dispatch: - inputs: - case_id: - type: choice - options: [gogs-moon, opencut-moon, stackstorm-pants, executorch-buck2, msgpack-sbt] - required: true - provider: - description: Selected provider for a native cache correctness screen. - type: choice - options: [all, boringcache, bazel-remote] - default: all - cli_version: - description: Exact CLI release or canary; required for BoringCache execution. - type: string - default: '' - series_id: - type: string - default: '' - sample: - type: string - default: '1' +--- +name: REAPI comparison fresh +run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample + }}" permissions: contents: read actions: read env: - BENCHMARK_SERIES_ID: ${{ inputs.series_id }} - BENCHMARK_SAMPLE: ${{ inputs.sample }} + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" BENCHMARK_RUNNER_CLASS: ubuntu-24.04 BORINGCACHE_NO_GIT: 'true' jobs: @@ -35,20 +15,21 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - matrix: ${{ steps.providers.outputs.matrix }} + matrix: "${{ steps.providers.outputs.matrix }}" steps: - - id: providers - env: - PROVIDER: ${{ inputs.provider }} - run: | - case "$PROVIDER" in - all) echo 'matrix=["boringcache","bazel-remote"]' >> "$GITHUB_OUTPUT" ;; - boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; - bazel-remote) echo 'matrix=["bazel-remote"]' >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + run: | + case "$PROVIDER" in + all) echo 'matrix=["boringcache","bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + bazel-remote) echo 'matrix=["bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac cold: - name: '${{ matrix.provider }} ${{ inputs.case_id }} cold' + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} cold" needs: providers runs-on: ubuntu-24.04 timeout-minutes: 180 @@ -59,24 +40,27 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/reapi-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: cold - cli-version: ${{ inputs.cli_version }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/reapi-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: cold + cli-version: "${{ inputs.cli_version }}" warm: - name: '${{ matrix.provider }} ${{ inputs.case_id }} warm' - needs: [providers, cold] + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} warm" + needs: + - providers + - cold runs-on: ubuntu-24.04 timeout-minutes: 180 permissions: @@ -86,18 +70,94 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/reapi-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: warm + cli-version: "${{ inputs.cli_version }}" + report: + name: "${{ inputs.case_id }} fresh report" + needs: + - cold + - warm + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/reapi-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: warm - cli-version: ${{ inputs.cli_version }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} fresh benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-fresh + path: benchmark-results/${{ inputs.case_id }}-boringcache-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain bazel-remote result + if: inputs.provider == 'all' || inputs.provider == 'bazel-remote' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-bazel-remote-fresh + path: benchmark-results/${{ inputs.case_id }}-bazel-remote-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-reapi-fresh + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - gogs-moon + - opencut-moon + - stackstorm-pants + - executorch-buck2 + - msgpack-sbt + required: true + provider: + description: Selected provider for a native cache correctness screen. + type: choice + options: + - all + - boringcache + - bazel-remote + default: all + cli_version: + description: Exact CLI release or canary; required for BoringCache execution. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' diff --git a/.github/workflows/reapi-rolling-benchmark.yml b/.github/workflows/reapi-rolling-benchmark.yml new file mode 100644 index 000000000..178ca1d4a --- /dev/null +++ b/.github/workflows/reapi-rolling-benchmark.yml @@ -0,0 +1,142 @@ +--- +name: REAPI comparison rolling +run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample + }}" +permissions: + contents: read + actions: read +env: + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" + BENCHMARK_RUNNER_CLASS: ubuntu-24.04 + BORINGCACHE_NO_GIT: 'true' + BENCHMARK_ROLLING_SCOPE: "${{ inputs.cache_scope }}" +jobs: + providers: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: "${{ steps.providers.outputs.matrix }}" + steps: + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + run: | + case "$PROVIDER" in + all) echo 'matrix=["boringcache","bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + bazel-remote) echo 'matrix=["bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + commit: + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} commit" + needs: providers + runs-on: ubuntu-24.04 + timeout-minutes: 180 + permissions: + contents: read + actions: read + id-token: write + strategy: + fail-fast: false + matrix: + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/reapi-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: commit + cli-version: "${{ inputs.cli_version }}" + lane: rolling + report: + name: "${{ inputs.case_id }} rolling report" + needs: + - commit + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} rolling benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-rolling + path: benchmark-results/${{ inputs.case_id }}-boringcache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain bazel-remote result + if: inputs.provider == 'all' || inputs.provider == 'bazel-remote' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-bazel-remote-rolling + path: benchmark-results/${{ inputs.case_id }}-bazel-remote-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-reapi-rolling + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +concurrency: + group: benchmark-reapi-rolling-${{ inputs.case_id }}-${{ inputs.cache_scope || inputs.series_id + || github.ref_name }} + cancel-in-progress: false + queue: max +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - gogs-moon + - opencut-moon + - stackstorm-pants + - executorch-buck2 + - msgpack-sbt + required: true + provider: + description: Selected provider for a native cache correctness screen. + type: choice + options: + - all + - boringcache + - bazel-remote + default: all + cli_version: + description: Exact CLI release or canary; required for BoringCache execution. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' + cache_scope: + type: string + default: '' diff --git a/.github/workflows/source-case.yml b/.github/workflows/source-case.yml new file mode 100644 index 000000000..bf9cef333 --- /dev/null +++ b/.github/workflows/source-case.yml @@ -0,0 +1,137 @@ +name: Check and queue one rolling case +on: + workflow_call: + inputs: + case_id: + type: string + required: true +permissions: + contents: read + actions: read +jobs: + previous: + name: '${{ inputs.case_id }} previous rolling run' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + actions: read + outputs: + proceed: ${{ steps.previous.outputs.proceed }} + ref: ${{ steps.previous.outputs.ref }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ vars.BENCHMARK_CADENCE_ACTIVE == 'true' && github.ref_name == 'main' && 'main' || github.sha }} + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - id: previous + env: + GH_TOKEN: ${{ github.token }} + CASE_ID: ${{ inputs.case_id }} + BENCHMARK_CADENCE_ACTIVE: ${{ vars.BENCHMARK_CADENCE_ACTIVE }} + run: | + if [[ "$BENCHMARK_CADENCE_ACTIVE" == true && "$GITHUB_REF_NAME" == main ]]; then + ruby scripts/source-promotion.rb --case "$CASE_ID" --reconcile \ + --output "$RUNNER_TEMP/rolling-previous.json" + else + echo 'proceed=true' >> "$GITHUB_OUTPUT" + echo "ref=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + fi + - name: Retain previous rolling state + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: rolling-previous-${{ inputs.case_id }} + path: ${{ runner.temp }}/rolling-previous.json + if-no-files-found: ignore + retention-days: 90 + inspect: + name: '${{ inputs.case_id }} source check' + needs: previous + if: needs.previous.outputs.proceed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + outputs: + changed: ${{ steps.proposal.outputs.changed }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ vars.BENCHMARK_CADENCE_ACTIVE == 'true' && github.ref_name == 'main' && 'main' || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Select the inspected revision from trusted history + env: + INSPECTED_REF: ${{ needs.previous.outputs.ref }} + run: | + [[ "$INSPECTED_REF" =~ ^[0-9a-f]{40}$ ]] || exit 1 + git merge-base --is-ancestor "$INSPECTED_REF" HEAD + git checkout --detach "$INSPECTED_REF" + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - name: Inspect upstream against the reviewed recipe + id: proposal + env: + GH_TOKEN: ${{ github.token }} + CASE_ID: ${{ inputs.case_id }} + run: | + ruby scripts/sync-sources.rb --case "$CASE_ID" --output "$RUNNER_TEMP/source-proposals.json" + echo "changed=$(ruby -rjson -e 'puts JSON.parse(File.read(ARGV[0])).fetch("records").first.fetch("state") == "proposal"' "$RUNNER_TEMP/source-proposals.json")" >> "$GITHUB_OUTPUT" + - name: Retain the source observation + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: source-proposals-${{ inputs.case_id }} + path: ${{ runner.temp }}/source-proposals.json + if-no-files-found: error + retention-days: 90 + publish: + name: '${{ inputs.case_id }} queue rolling runs' + needs: [previous, inspect] + if: needs.inspect.outputs.changed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + actions: write + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ vars.BENCHMARK_CADENCE_ACTIVE == 'true' && github.ref_name == 'main' && 'main' || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Select the inspected revision from trusted history + env: + INSPECTED_REF: ${{ needs.previous.outputs.ref }} + run: | + [[ "$INSPECTED_REF" =~ ^[0-9a-f]{40}$ ]] || exit 1 + git merge-base --is-ancestor "$INSPECTED_REF" HEAD + git checkout --detach "$INSPECTED_REF" + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: source-proposals-${{ inputs.case_id }} + path: ${{ runner.temp }}/proposal + - name: Publish source and request the declared rolling variants + env: + GH_TOKEN: ${{ github.token }} + CASE_ID: ${{ inputs.case_id }} + BENCHMARK_CADENCE_ACTIVE: ${{ vars.BENCHMARK_CADENCE_ACTIVE }} + run: | + args=(--channel canary) + if [[ "$BENCHMARK_CADENCE_ACTIVE" == true && "$GITHUB_REF_NAME" == main ]]; then + args=(--channel stable --publish) + fi + ruby scripts/source-promotion.rb --case "$CASE_ID" "${args[@]}" \ + --proposal "$RUNNER_TEMP/proposal/source-proposals.json" \ + --output "$RUNNER_TEMP/rolling-dispatch.json" + - name: Retain rolling dispatch receipt + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: rolling-dispatch-${{ inputs.case_id }} + path: ${{ runner.temp }}/rolling-dispatch.json + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/source-sync.yml b/.github/workflows/source-sync.yml index cfe31321a..c13b89116 100644 --- a/.github/workflows/source-sync.yml +++ b/.github/workflows/source-sync.yml @@ -1,43 +1,52 @@ -name: Propose Upstream Source Updates -on: - workflow_dispatch: - inputs: - case_id: - description: Optional case ID; empty inspects all maintained source schedules - type: string - default: '' +--- +name: Check upstream source updates permissions: contents: read actions: read -concurrency: - group: benchmark-source-proposals - cancel-in-progress: false jobs: - propose: + select: + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE + == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 10 + outputs: + matrix: "${{ steps.cases.outputs.matrix }}" steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - run: bundle install --jobs 4 --retry 2 - - name: Prepare reviewed source proposals - env: - GH_TOKEN: ${{ github.token }} - CASE_ID: ${{ inputs.case_id }} - run: | - args=() - [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") - ruby scripts/sync-sources.rb "${args[@]}" --output "$RUNNER_TEMP/source-proposals.json" - - name: Retain proposed definitions and qualification requirements - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f - with: - name: source-proposals - path: | - ${{ runner.temp }}/source-proposals.json - cases/*/case.json - cases/*/payload/benchmark-source.env - if-no-files-found: error - retention-days: 90 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - id: cases + env: + CASE_ID: "${{ inputs.case_id }}" + run: | + args=() + [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") + echo "matrix=$(ruby scripts/sync-sources.rb --matrix "${args[@]}")" >> "$GITHUB_OUTPUT" + sources: + name: "${{ matrix.case_id }}" + needs: select + permissions: + contents: write + actions: write + concurrency: + group: benchmark-source-${{ matrix.case_id }} + cancel-in-progress: false + queue: max + strategy: + fail-fast: false + max-parallel: 6 + matrix: "${{ fromJSON(needs.select.outputs.matrix) }}" + uses: "./.github/workflows/source-case.yml" + with: + case_id: "${{ matrix.case_id }}" +'on': + schedule: + - cron: 11 * * * * + workflow_dispatch: + inputs: + case_id: + description: Optional case ID; empty checks the scheduled suite. + type: string + default: '' diff --git a/.github/workflows/weekly-fresh.yml b/.github/workflows/weekly-fresh.yml new file mode 100644 index 000000000..556ee8bcf --- /dev/null +++ b/.github/workflows/weekly-fresh.yml @@ -0,0 +1,28 @@ +name: Weekly fresh benchmarks +on: + schedule: + - cron: '0 4 * * 1' + workflow_dispatch: + inputs: + cli_version: + description: Optional exact stable CLI release; empty selects the latest. + type: string + default: '' + dry_run: + description: Validate the dispatch without starting builds. + type: boolean + default: false +permissions: + contents: read + actions: write +jobs: + dispatch: + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' + concurrency: + group: benchmark-weekly-dispatch + cancel-in-progress: false + uses: ./.github/workflows/fresh-cadence.yml + with: + channel: stable + cli_version: ${{ inputs.cli_version || '' }} + dry_run: ${{ inputs.dry_run || false }} diff --git a/.github/workflows/zed-zed-cargo-rolling-auto.yml b/.github/workflows/zed-zed-cargo-rolling-auto.yml index 6c0b2f5e8..216471156 100644 --- a/.github/workflows/zed-zed-cargo-rolling-auto.yml +++ b/.github/workflows/zed-zed-cargo-rolling-auto.yml @@ -166,7 +166,7 @@ jobs: permissions: actions: read checks: read - contents: write + contents: read id-token: write publish-source: name: Publish the verified Zed source diff --git a/cases/chroma/case.json b/cases/chroma/case.json index eb47dab4d..0abc6c3d8 100644 --- a/cases/chroma/case.json +++ b/cases/chroma/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/chroma-docker-benchmark", "benchmark_id": "chroma", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/duckgres/case.json b/cases/duckgres/case.json index 0e46cc0a7..a6180fa11 100644 --- a/cases/duckgres/case.json +++ b/cases/duckgres/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/duckgres-docker-benchmark", "benchmark_id": "duckgres", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/executorch-buck2/case.json b/cases/executorch-buck2/case.json index 068382899..c0ebad736 100644 --- a/cases/executorch-buck2/case.json +++ b/cases/executorch-buck2/case.json @@ -22,7 +22,7 @@ "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", @@ -30,6 +30,13 @@ "inputs": { "case_id": "executorch-buck2" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } } ], "blockers": [] @@ -51,9 +58,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/gogs-moon/case.json b/cases/gogs-moon/case.json index 8c113a24b..de759e8ff 100644 --- a/cases/gogs-moon/case.json +++ b/cases/gogs-moon/case.json @@ -21,7 +21,7 @@ "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", @@ -29,6 +29,13 @@ "inputs": { "case_id": "gogs-moon" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } } ], "blockers": [] @@ -50,9 +57,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/gogs-moon/payload/recipe-contract.json b/cases/gogs-moon/payload/recipe-contract.json index dfb80366b..148157c28 100644 --- a/cases/gogs-moon/payload/recipe-contract.json +++ b/cases/gogs-moon/payload/recipe-contract.json @@ -6,5 +6,15 @@ "pnpm-lock.yaml": "eb8bbc6812fbdfb3c847d99d4af09925e9b4c6b285e99472c4193d483d90f523", "web/package.json": "e8e4ea83c1bdb84effa1ec14c45d63024e0f13b197a36847b8680dfd0b57aeb4" }, - "commands": {} + "commands": { + }, + "upstream_files_by_revision": { + "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b": { + ".moon/workspace.yml": "74f75a4b2cc84d452c7911159396fc28ec858e67c6033a54ec30baa12696d73f", + "moon.yml": "e2c40b021b55a6a0c5de5c1de90bf659471cacfdedf684dd318ebfac74ff2616", + "web/moon.yml": "e800f64593003a6cdb5cd0cce17da3617b811bb712eb8263731e02de3a164cd2", + "pnpm-lock.yaml": "801e39410e1ee70f9d1b5872d17fc6e7b103fe1b6335d185b5d6cb8d61e3df2b", + "web/package.json": "01e0ab93e1abf863194730dd6ea4bf84c3c7ee1a0b4cd89ec098956528f0a2ac" + } + } } diff --git a/cases/helix-nix/case.json b/cases/helix-nix/case.json index bb314e586..aa1dc8231 100644 --- a/cases/helix-nix/case.json +++ b/cases/helix-nix/case.json @@ -21,7 +21,7 @@ "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/nix-fresh-benchmark.yml", @@ -30,6 +30,13 @@ "case_id": "helix-nix", "provider": "all" } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } } ], "blockers": [], @@ -55,9 +62,9 @@ "cachix" ], "primary_metric": "build_and_reuse_seconds", - "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", "storage": "provider-reported", - "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/hugo/case.json b/cases/hugo/case.json index 1c483b0f9..99b8ffe30 100644 --- a/cases/hugo/case.json +++ b/cases/hugo/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/hugo-docker-benchmark", "benchmark_id": "hugo", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/immich/case.json b/cases/immich/case.json index 057bc2dfa..9c07a745e 100644 --- a/cases/immich/case.json +++ b/cases/immich/case.json @@ -33,6 +33,13 @@ "path": ".github/workflows/immich-immich-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } } ], "sync": "upstream-head", @@ -47,7 +54,9 @@ "fresh_inputs": { "load_image": "true" }, - "rolling_inputs": {} + "rolling_inputs": { + "load_image": "true" + } } }, "reporting": { diff --git a/cases/immich/payload/recipe-contract.json b/cases/immich/payload/recipe-contract.json index c8f263b77..857b05ebd 100644 --- a/cases/immich/payload/recipe-contract.json +++ b/cases/immich/payload/recipe-contract.json @@ -1,8 +1,8 @@ { "schema_version": 1, "upstream_files": { - ".github/workflows/docker.yml": "79f9b93dc0bdfc5c9e55550331522837329ce061bebfa5737d97a27ea3b4b795", - "server/Dockerfile": "b64d6daffd88147d57fae1ff115fe92f520b6df9a6f33a8da66648f80a0821da" + ".github/workflows/docker.yml": "2545e683c5bb11ab0e2ab9a706289c0603be5ded38a560dc6faf57528dca5cd9", + "server/Dockerfile": "fd0231bdac647f70c31caf0ff22a23bbfd8fb74666d3d52cb25d8ea8fd819757" }, "commands": { "docker": [ @@ -27,5 +27,11 @@ "immich-server-benchmark:local", "upstream" ] + }, + "upstream_files_by_revision": { + "007c18aec6cc9552fb7d8b7c791c41d7424876b0": { + ".github/workflows/docker.yml": "79f9b93dc0bdfc5c9e55550331522837329ce061bebfa5737d97a27ea3b4b795", + "server/Dockerfile": "b64d6daffd88147d57fae1ff115fe92f520b6df9a6f33a8da66648f80a0821da" + } } } diff --git a/cases/linkerd2/case.json b/cases/linkerd2/case.json index 66a8ff1f5..f18f2ad84 100644 --- a/cases/linkerd2/case.json +++ b/cases/linkerd2/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/linkerd2-docker-benchmark", "benchmark_id": "linkerd2-v2", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/mastodon/case.json b/cases/mastodon/case.json index 08c376a67..e85f408c0 100644 --- a/cases/mastodon/case.json +++ b/cases/mastodon/case.json @@ -39,6 +39,19 @@ "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" } ], "sync": "upstream-head", @@ -55,7 +68,8 @@ "load_image": "true" }, "rolling_inputs": { - "workload": "server" + "workload": "server", + "load_image": "true" }, "variants": { "server": {}, diff --git a/cases/msgpack-sbt/case.json b/cases/msgpack-sbt/case.json index 30c742ae4..07e6d7fae 100644 --- a/cases/msgpack-sbt/case.json +++ b/cases/msgpack-sbt/case.json @@ -29,9 +29,16 @@ "inputs": { "case_id": "msgpack-sbt" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } } ], - "sync": "fixed", + "sync": "upstream-head", "blockers": [] }, "reporting": { @@ -50,10 +57,10 @@ "boringcache", "bazel-remote" ], - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "primary_metric": "correctness", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/msgpack-sbt/payload/recipe-contract.json b/cases/msgpack-sbt/payload/recipe-contract.json index 3b9f981bd..e75a08504 100644 --- a/cases/msgpack-sbt/payload/recipe-contract.json +++ b/cases/msgpack-sbt/payload/recipe-contract.json @@ -1,9 +1,18 @@ { "upstream_files": { "build.sbt": "c4e03e280c4da5cdc930759bf7fdd368c5d30999290aded4cd636042ebc0e89c", - "project/build.properties": "331ebccd1dbd48154f6c230393795dd72d370e6fdb9952ff2dd22b9eae4902b2", + "project/build.properties": "94a37bc3ad2e038979be0b7172c35904a468ab8e6e9c61e0993dd95af77b23ce", "project/plugins.sbt": "eab49425b64f57fcdfaef54f4ec1722e78165a86e51ece33751566adcb7db6ac", "sbt": "b7cd1f08b549fc67528c9d6a6a026a3342e82ba46dc5dc8a98729ceb201d0581" }, - "commands": {} + "commands": { + }, + "upstream_files_by_revision": { + "5c28fbad6cd360d8ea822df109830ae2517a7ae4": { + "build.sbt": "c4e03e280c4da5cdc930759bf7fdd368c5d30999290aded4cd636042ebc0e89c", + "project/build.properties": "331ebccd1dbd48154f6c230393795dd72d370e6fdb9952ff2dd22b9eae4902b2", + "project/plugins.sbt": "eab49425b64f57fcdfaef54f4ec1722e78165a86e51ece33751566adcb7db6ac", + "sbt": "b7cd1f08b549fc67528c9d6a6a026a3342e82ba46dc5dc8a98729ceb201d0581" + } + } } diff --git a/cases/n8n/case.json b/cases/n8n/case.json index 2aad3cfd2..d03785284 100644 --- a/cases/n8n/case.json +++ b/cases/n8n/case.json @@ -45,6 +45,20 @@ "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" } ], "sync": "upstream-head", @@ -74,7 +88,8 @@ }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", - "image": "n8n-benchmark" + "image": "n8n-benchmark", + "load_image": "true" } }, "runners": { @@ -87,7 +102,8 @@ }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile", - "image": "n8n-runners-benchmark" + "image": "n8n-runners-benchmark", + "load_image": "true" } }, "distroless": { @@ -100,7 +116,8 @@ }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", - "image": "n8n-runners-distroless-benchmark" + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" } } } diff --git a/cases/obs-studio/case.json b/cases/obs-studio/case.json index b4daea835..8f89c2025 100644 --- a/cases/obs-studio/case.json +++ b/cases/obs-studio/case.json @@ -30,7 +30,8 @@ "path": ".github/workflows/obs-studio-obs-actions-cache.yml", "variant_input": "cache_tool", "lane": "fresh", - "inputs": {}, + "inputs": { + }, "phases": [ "cold", "commit" @@ -44,7 +45,8 @@ "path": ".github/workflows/obs-studio-obs-boringcache.yml", "variant_input": "cache_tool", "lane": "fresh", - "inputs": {}, + "inputs": { + }, "phases": [ "cold", "commit" @@ -57,15 +59,33 @@ { "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", "lane": "diagnostic", - "inputs": {} + "inputs": { + } + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": { + }, + "variants": [ + "ccache", + "xcode" + ] } ], - "sync": "adjacent-pair", + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, "sync_minute": 18, "source_prefix": "OBS" }, "reporting": { - "entries": [], + "entries": [ + + ], "publication": "reviewed" }, "verification": [ diff --git a/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb b/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb index b537f232f..7169b4661 100755 --- a/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb +++ b/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb @@ -7,14 +7,19 @@ %w[path phase strategy].each { |name| parser.on("--#{name} VALUE") { |value| args[name] = value } } end.parse! payload = JSON.parse(File.read(args.fetch("path"))) -raise "Unknown phase or strategy" unless %w[base rolling].include?(args["phase"]) && %w[actions-cache boringcache].include?(args["strategy"]) +raise "Unknown phase or strategy" unless %w[base rolling commit].include?(args["phase"]) && %w[actions-cache boringcache].include?(args["strategy"]) errors, timeouts = payload.values_at("remote_storage_error", "remote_storage_timeout").map(&:to_i) raise "ccache reported remote errors=#{errors}, timeouts=#{timeouts}" unless errors.zero? && timeouts.zero? hits = payload.fetch("direct_cache_hit", 0) + payload.fetch("preprocessed_cache_hit", 0) base = args["phase"] == "base" raise "Cold ccache build did not report cache misses" if base && payload.fetch("cache_miss", 0) <= 0 -raise "Rolling ccache build did not report cache hits" if !base && hits <= 0 +raise "Rolling ccache build did not report cache hits" if args["phase"] == "rolling" && hits <= 0 +raise "ccache build did not report compilation" if args["phase"] == "commit" && hits + payload.fetch("cache_miss", 0) <= 0 if args["strategy"] == "boringcache" - field = base ? "remote_storage_write" : "remote_storage_hit" - raise "BoringCache did not report #{field}" if payload.fetch(field, 0) <= 0 + if args["phase"] == "commit" + raise "BoringCache did not report a remote read or write" if payload.fetch("remote_storage_hit", 0) + payload.fetch("remote_storage_write", 0) <= 0 + else + field = base ? "remote_storage_write" : "remote_storage_hit" + raise "BoringCache did not report #{field}" if payload.fetch(field, 0) <= 0 + end end diff --git a/cases/obs-studio/payload/scripts/write_phase_result.rb b/cases/obs-studio/payload/scripts/write_phase_result.rb index 9554e7aae..5ef340509 100755 --- a/cases/obs-studio/payload/scripts/write_phase_result.rb +++ b/cases/obs-studio/payload/scripts/write_phase_result.rb @@ -6,12 +6,16 @@ args = {"output_dir" => "benchmark-results", "cache_key" => ""} OptionParser.new do |parser| - %w[surface strategy phase source_sha restore_seconds build_seconds cache_key action_evidence native_evidence output_dir].each do |name| + %w[surface strategy phase source_sha restore_seconds build_seconds cache_key cache_hit action_evidence native_evidence output_dir].each do |name| parser.on("--#{name.tr('_','-')} VALUE") { |value| args[name] = value } end + parser.on("--continuous") { args["continuous"] = true } end.parse! phase = args.fetch("phase") raise "Phase must be base or rolling" unless %w[base rolling].include?(phase) +raise "Continuous observations require the rolling phase" if args["continuous"] && phase != "rolling" +cache_hit = args["continuous"] ? args["cache_hit"] : (phase == "base" ? "false" : "true") +raise "Cache hit must be true or false" if cache_hit && !%w[true false].include?(cache_hit) build_seconds = Float(args.fetch("build_seconds")) raise "Build time must be finite and nonnegative" unless build_seconds.finite? && build_seconds >= 0 native = args["native_evidence"] && JSON.parse(File.read(args["native_evidence"])) @@ -31,10 +35,14 @@ "remote_storage_hits" => native.fetch("remote_storage_hit", 0), "remote_storage_writes" => native.fetch("remote_storage_write", 0), "remote_storage_errors" => native.fetch("remote_storage_error", 0), "remote_storage_timeouts" => native.fetch("remote_storage_timeout", 0)}, "github" => BenchmarkReport.github_identity} +if args["continuous"] + result["classification"]["reporting_reason"] = "Source build using the persistent compiler-cache cohort" + result["classification"]["cache_import_status"] = {"true" => "hit", "false" => "miss"}.fetch(cache_hit, "unmeasured") +end BenchmarkReport.write_json(File.join(args["output_dir"], "#{args['surface']}-#{args['strategy']}-#{phase}.json"), result) -BenchmarkReport.phase({"benchmark" => "obs-studio-#{args['surface']}", "strategy" => args["strategy"], "variant" => args.fetch("surface"), +BenchmarkReport.phase({"benchmark" => args["continuous"] ? "obs-studio" : "obs-studio-#{args['surface']}", "strategy" => args["strategy"], "variant" => args.fetch("surface"), "lane" => phase == "base" ? "fresh" : "rolling", "phase" => phase == "base" ? "cold" : "commit", "mode" => args["surface"], "source_repository" => "obsproject/obs-studio", "source_sha" => args["source_sha"], "build_seconds" => build_seconds, - "restore_or_setup_seconds" => Integer(args["restore_seconds"]), "cache_hit" => phase == "base" ? "false" : "true", + "restore_or_setup_seconds" => Integer(args["restore_seconds"]), "cache_hit" => cache_hit, "evidence" => args["action_evidence"], "storage_key" => args["cache_key"], "output_dir" => args["output_dir"], "verification_passed" => true, "comparison_seconds" => Integer(args["restore_seconds"]) + build_seconds}) diff --git a/cases/opencut-moon/case.json b/cases/opencut-moon/case.json index 8b2023d12..1471e64a6 100644 --- a/cases/opencut-moon/case.json +++ b/cases/opencut-moon/case.json @@ -21,7 +21,7 @@ "revision": "e668010778568641babef2cc40be4703ae6916d6" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", @@ -29,6 +29,13 @@ "inputs": { "case_id": "opencut-moon" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } } ], "blockers": [] @@ -50,9 +57,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/posthog/case.json b/cases/posthog/case.json index bfddc7ca3..dc86a3a28 100644 --- a/cases/posthog/case.json +++ b/cases/posthog/case.json @@ -38,6 +38,18 @@ "path": ".github/workflows/posthog-posthog-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" } ], "sync": "upstream-head", @@ -54,7 +66,8 @@ "load_image": "true" }, "rolling_inputs": { - "platform": "linux/amd64" + "platform": "linux/amd64", + "load_image": "true" }, "variants": { "layers": {}, diff --git a/cases/qdrant/payload/recipe-contract.json b/cases/qdrant/payload/recipe-contract.json index f5ea1ba36..707ff2586 100644 --- a/cases/qdrant/payload/recipe-contract.json +++ b/cases/qdrant/payload/recipe-contract.json @@ -1,7 +1,7 @@ { "schema_version": 1, "upstream_files": { - ".github/workflows/integration-tests.yml": "b8ce312d966388b4ea8b748eedc9ad61d15934d684ae06e4cf29b546a5084418", + ".github/workflows/integration-tests.yml": "ca7026d1a1e15e45bd40cf92dbc4a44828b374e3fae71cbf64fc70d830c871cf", "Dockerfile": "e5796337f4c1fa080ddad9dd4260e5b8eec9dc6920ded0b4a3332e73bfb1db7f" }, "commands": { @@ -20,5 +20,11 @@ "qdrant/qdrant:e2e-tests", "upstream" ] + }, + "upstream_files_by_revision": { + "9ae05efdc1ef96352889b2a6b6b574c35232b974": { + ".github/workflows/integration-tests.yml": "b8ce312d966388b4ea8b748eedc9ad61d15934d684ae06e4cf29b546a5084418", + "Dockerfile": "e5796337f4c1fa080ddad9dd4260e5b8eec9dc6920ded0b4a3332e73bfb1db7f" + } } } diff --git a/cases/stackstorm-pants/case.json b/cases/stackstorm-pants/case.json index ff36584be..1cc500ef1 100644 --- a/cases/stackstorm-pants/case.json +++ b/cases/stackstorm-pants/case.json @@ -21,7 +21,7 @@ "revision": "9824de4dfd0c869869e310dee729308f398ad83a" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", @@ -29,6 +29,13 @@ "inputs": { "case_id": "stackstorm-pants" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } } ], "blockers": [] @@ -50,9 +57,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/zed-nix/case.json b/cases/zed-nix/case.json index 409ba47d6..87058a727 100644 --- a/cases/zed-nix/case.json +++ b/cases/zed-nix/case.json @@ -21,7 +21,7 @@ "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/nix-fresh-benchmark.yml", @@ -30,6 +30,13 @@ "case_id": "zed-nix", "provider": "all" } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } } ], "blockers": [], @@ -55,9 +62,9 @@ "cachix" ], "primary_metric": "build_and_reuse_seconds", - "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", "storage": "provider-reported", - "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/zed/case.json b/cases/zed/case.json index 57eb38fd4..b5c4d0409 100644 --- a/cases/zed/case.json +++ b/cases/zed/case.json @@ -76,6 +76,7 @@ } ], "sync": "verified-pair", + "rolling_workflow": ".github/workflows/zed-zed-cargo-rolling-auto.yml", "sync_minute": 28, "source_prefix": "ZED", "cache_tags": { diff --git a/cases/zed/payload/recipe-contract.json b/cases/zed/payload/recipe-contract.json index c723c1b25..8add934c2 100644 --- a/cases/zed/payload/recipe-contract.json +++ b/cases/zed/payload/recipe-contract.json @@ -2,7 +2,7 @@ "schema_version": 1, "upstream_files": { "rust-toolchain.toml": "3d9002e97d5c3de93d49a1a36649153da4f40c29b061d0f2e09ea84ee143ba3e", - "script/bundle-linux": "6c8ceb443cb364259aeadebbfb47dba829cef81fccef304bd19d20724f1f0be7" + "script/bundle-linux": "8d269b2d6f304440f3a0951d5dfd79837c08fb2cf2bd5e8fbf59bb589428e43d" }, "commands": { "cargo": [ @@ -11,5 +11,11 @@ "--release", "--locked" ] + }, + "upstream_files_by_revision": { + "54edefa0832e4728bcf077e4a48b25912eea5433": { + "rust-toolchain.toml": "3d9002e97d5c3de93d49a1a36649153da4f40c29b061d0f2e09ea84ee143ba3e", + "script/bundle-linux": "6c8ceb443cb364259aeadebbfb47dba829cef81fccef304bd19d20724f1f0be7" + } } } diff --git a/docs/cadence.md b/docs/cadence.md new file mode 100644 index 000000000..7f891dcbb --- /dev/null +++ b/docs/cadence.md @@ -0,0 +1,105 @@ +# Benchmark schedules + +Central activation remains off. Historical repositories still own the active +weekly, canary and source schedules. Central scheduled jobs require +`BENCHMARK_CADENCE_ACTIVE=true`; merging this branch does not activate them. +Manual qualification remains available. Historical triggers must be retired +before ownership changes, so the migration does not duplicate runs. + +`suites/scheduled.json` selects the maintained suite plus Moon, Pants, Buck2, +sbt and Nix: 24 source cases and 31 fresh targets. It excludes Docker corpus and +prospect drafts. Case definitions own workflow paths and variant inputs. + +| Run | UTC schedule | CLI | Source | +| --- | --- | --- | --- | +| Weekly fresh | Monday 04:00 | Latest compatible published stable release | Declared pins | +| Nightly fresh | Daily 01:17 | Latest published canary | Declared pins | +| Rolling source check | Hourly, at :11 | Stable after activation; canary for pre-activation planning | Reviewed upstream candidates | + +## Independent dispatch and ordered rolling caches + +Fresh dispatch validates the whole suite and resolves one exact CLI tag before +starting a matrix of independent requests. Each target retains its own receipt. +The combined receipt preserves requested run IDs, failed or uncertain requests, +and missing targets. A dispatch succeeding does not establish that the builds +passed. Dry runs request no builds. Interrupted requests require receipt review +before retrying; an absent HTTP response does not prove GitHub rejected a run. + +Source checks fan out by case, with six checks running at a time. Each case owns +its source-controller lock; one failed or slow case does not stop the others. +Rolling workflows serialize publication to the same cache with `queue: max`. +The next source cycle inspects the previous requested runs before advancing that +case again. An unchanged source does not request another build. + +Source inspection runs with read-only repository permissions. A separate job +reconstructs only permitted source-pin changes, verifies the expected case and +shared harness, then conditionally publishes against the current branch head. +It records dispatch intent before making requests and retains receipts under +`migration/rolling/`, including an observation per candidate source. A concurrent +edit to the same case or shared harness blocks publication; an unrelated case's +commit can be retried against its new branch head. Publication uses GitHub's +[conditional commit API](https://docs.github.com/en/graphql/reference/commits#createcommitonbranch). + +Deno retains adjacent-commit order. OBS retains its existing selection of the +next build-relevant change and its immediate parent, recording the previous +source separately. Zed selects dependency-verified candidates and promotes a +source only after its rolling build succeeds. Other cases retain their existing +promote-then-build order. Failed builds remain observations. Uncertain requests +block that case from automatic redispatch until their receipts are reviewed. + +Before activation, the publication job only plans with a canary. It does not +change `main` or request rolling builds. Both the workflow and publisher require +active central ownership on `main` before publication. + +## Workloads and reports + +Every scheduled case now registers a rolling path, including the four native +cases, OBS, five REAPI cases and two Nix cases previously missing one. Definitions +and passing local checks do not establish live rolling qualification. + +Native Docker runs load and inspect their output for both providers. REAPI +rolling runs retain the case/series cache; bazel-remote restores and publishes a +GitHub Actions cache and records the previous source and restored key. Nix and +OBS rolling runs permit both restore and publication. Missing seeds remain +bootstrap observations; they must not be described as proven cache reuse. + +Nix workers import one common dependency store prepared outside the measured +build. The export excludes the measured package, checksums the archive and +checks the imported dependency hashes against the declared derivation. This +avoids independently rebuilding a nondeterministic dependency for each arm while +keeping strict package and dependency comparisons. The earlier Zed Nix screen +restored both providers' package outputs correctly but failed the dependency +baseline comparison; that failed run remains retained. + +Shared workflows use case, variant, lane, provider and phase labels. REAPI, Nix +and OBS rolling results use the canonical benchmark report and retain both +structured evidence and the comparison summary. Reports contain measurements +and verification results, without performance verdicts. + +## Qualification and activation + +The October 5 preflight rejected stable `v1.33.0`: it does not expose +`cache-registry --reapi-port`. Canary `vcli-canary-7a5b27146ebe` passed the full +31-target preflight. Weekly activation must wait for a compatible stable release; +it must not substitute a canary or silently omit the REAPI cases. + +The first hosted source matrix completed the cases independently and identified +recipe changes in Immich, Qdrant, msgpack and Zed. Their reviewed changes are +recorded in [the recipe review](recipe-reviews-2026-10-05.md). New source pins and +changed recipes still require live qualification. + +The repository's Actions cache limit is 200 GB, with seven-day retention. +Capacity alone does not establish that a particular rolling seed was retained +or restored. + +Remaining activation requirements: + +- Pass hosted checks for the complete controller and rolling changes. +- Qualify the new and changed rolling paths with actual source advancement, + retained seeds, verified outputs and canary CLI execution. +- Pass the corrected Zed Nix cold/warm screen using the common dependency seed. +- Verify publication and receipt reconciliation in a bounded rehearsal. +- Retire replaced historical cron triggers while preserving manual and release + entrypoints; verify no outstanding duplicate dispatches. +- After a compatible stable release, change monitoring and publication ownership + and set `BENCHMARK_CADENCE_ACTIVE=true`. diff --git a/docs/process.md b/docs/process.md index 5dd65cec6..265662824 100644 --- a/docs/process.md +++ b/docs/process.md @@ -81,6 +81,9 @@ only when the experiment requires a different shape, and register it in the case Use the canonical Ruby reporter for every execution path. Add suite membership for an explicit scheduling purpose. `suites/published.json` is the shared registry for published reporting. +`suites/scheduled.json` selects the maintained suite and new native families for +the proposed shared cadence. See [benchmark schedules](cadence.md) for current +ownership and the remaining source and rolling work. `.github/actions/boringcache` is the sole BoringCache One invocation and release pin. It forwards the product inputs used by the cases and retains raw evidence. diff --git a/docs/recipe-reviews-2026-10-05.md b/docs/recipe-reviews-2026-10-05.md new file mode 100644 index 000000000..513424b37 --- /dev/null +++ b/docs/recipe-reviews-2026-10-05.md @@ -0,0 +1,24 @@ +# Upstream recipe review, October 5 + +The hourly source check at [run 37315116759](https://github.com/boringcache/benchmarks/actions/runs/37315116759) +identified four recipe changes. These reviews approve the recipe definitions for +qualification; they do not establish that a new-source build or cache restore passed. + +| Case | Reviewed candidate | Change and measurement impact | +| --- | --- | --- | +| Immich | `f938b2faa77cc25064c2ce54c956c40e4dd4ee79` | The Docker workflow adds a machine-learning OpenVINO runner mapping, outside the server workload. The server Dockerfile advances both pinned base images from 202609291109 to 202610021601. The server command and output check remain the same; new base images require a new build observation. | +| Qdrant | `48199a3c2d4156d0ab7e692f3925a9ed14059008` on `dev` | Integration jobs now download one shared debug binary and split consensus tests into shards. E2E jobs build one Docker image, export it to an artifact and load it on test workers. The Dockerfile and build arguments are unchanged. The benchmark retains local image loading and verification; artifact transfer and test sharding remain outside its build measurement. | +| msgpack | `ec75ffc01364adc6ea9b5f4d9129fd63602e6a45` | `project/build.properties` advances sbt 2.0.9 to 2.0.10. `build.sbt`, `project/plugins.sbt` and the launcher are unchanged. Native REAPI execution must be screened with the new sbt version. | +| Zed | `dff535449c663f3e2fde9256db2620c2ace2bc4c` | The release packaging script changes Sentry handling, linker flags and stripping of packaged copies. The rolling workload remains its declared `cargo build --release --locked`; it does not measure the release packaging script. The separately pinned release-layer plans and their recipe contract remain unchanged. | + +Each contract contains the new reviewed file hashes. Its existing parent source +can still be verified using `upstream_files_by_revision`, keyed by its complete +commit SHA. That exception retains the same reviewed file set and does not apply +to other revisions. Source advancement still requires recipe verification; a +later change to any reviewed file blocks advancement again. + +The Gogs rolling qualification also permits the exact historical parent +`103893ed5f78d2f138f5286a2dd3b4e94d18fb0b`. Its web manifest uses `@pierre/diffs` +1.2.3; the declared child advances it to 1.4.1 with matching lockfile updates. +The Moon build commands are unchanged. These are changed-source observations, +including changed dependencies, rather than identical-source replay observations. diff --git a/migration/cache-capacity-2026-10-05.json b/migration/cache-capacity-2026-10-05.json new file mode 100644 index 000000000..bca710ed1 --- /dev/null +++ b/migration/cache-capacity-2026-10-05.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "scope": "Repository capacity after the approved limit increase. These observations do not establish per-phase seed availability or restoration.", + "observations": [ + { + "collected_at": "2026-10-05T12:37:50Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/storage-limit", + "response": { + "max_cache_size_gb": 200 + } + }, + { + "collected_at": "2026-10-05T12:37:51Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/retention-limit", + "response": { + "max_cache_retention_days": 7 + } + }, + { + "collected_at": "2026-10-05T12:37:51Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/usage", + "response": { + "full_name": "boringcache/benchmarks", + "active_caches_size_in_bytes": 10135681922, + "active_caches_count": 262 + } + } + ] +} diff --git a/results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json b/results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..220e76d8d --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T13:57:41Z", + "ref": "shared-benchmark-cadence", + "state": "requested", + "run_id": 37320867908, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320867908" +} diff --git a/results/helix-nix/cadence-fresh-01/series.json b/results/helix-nix/cadence-fresh-01/series.json new file mode 100644 index 000000000..417c0d9b0 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "created_at": "2026-10-05T13:54:56Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "c2a0ba9332802d28a66f0938b4fd31a43ff928f6d8e7348f2f07f37e1c933421" +} diff --git a/results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json b/results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..abfc0f3f0 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T13:57:42Z", + "ref": "shared-benchmark-cadence", + "state": "requested", + "run_id": 37320869828, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320869828" +} diff --git a/results/zed-nix/cadence-fresh-01/series.json b/results/zed-nix/cadence-fresh-01/series.json new file mode 100644 index 000000000..1f6e46187 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "created_at": "2026-10-05T13:54:56Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "variant": null, + "workflow_inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "97e97ded96f1686fba28581b92a015bace71e0fcaf38d7ee82235e290ab4f7cd" +} diff --git a/results/zed-nix/canary-screening-02/completions/37296214039.json b/results/zed-nix/canary-screening-02/completions/37296214039.json new file mode 100644 index 000000000..482cb44bf --- /dev/null +++ b/results/zed-nix/canary-screening-02/completions/37296214039.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296214039", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "cachix zed-nix warm: failure", + "boringcache zed-nix warm: failure" + ], + "evidence_manifest_sha256": "3a2c88abb871648fc13d495f49cd9476f0b6c20045d2f4828c2bf6203d60d1fd" +} diff --git a/results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json b/results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..0b1948f1b --- /dev/null +++ b/results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "canary-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T10:22:54Z", + "ref": "run-tool-family-cases", + "state": "requested", + "run_id": 37296214039, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039" +} diff --git a/results/zed-nix/canary-screening-02/report.json b/results/zed-nix/canary-screening-02/report.json new file mode 100644 index 000000000..cfa796cd4 --- /dev/null +++ b/results/zed-nix/canary-screening-02/report.json @@ -0,0 +1,424 @@ +{ + "schema_version": 1, + "plan_sha256": "232baf4cadd9705cb853cd1410c1724fbb6b3869458cfa2bb489ea9148570a77", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296214039", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "cachix zed-nix warm: failure", + "boringcache zed-nix warm: failure" + ], + "evidence_manifest_sha256": "3a2c88abb871648fc13d495f49cd9476f0b6c20045d2f4828c2bf6203d60d1fd" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296214039", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "cachix zed-nix warm: failure", + "boringcache zed-nix warm: failure" + ], + "evidence_manifest_sha256": "3a2c88abb871648fc13d495f49cd9476f0b6c20045d2f4828c2bf6203d60d1fd" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "cachix" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 2788.14372093361, + "min": 2788.14372093361, + "max": 2788.14372093361 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 2996.7530764946086, + "min": 2996.7530764946086, + "max": 2996.7530764946086 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3193912506103516, + "build_seconds": 2785.8243296829996, + "workflow_seconds": null, + "total_seconds": 2788.14372093361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2788.14372093361 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173730", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.6468822956085205, + "build_seconds": 2994.106194199, + "workflow_seconds": null, + "total_seconds": 2996.7530764946086, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2996.7530764946086 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173731", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/canary-screening-02/report.md b/results/zed-nix/canary-screening-02/report.md new file mode 100644 index 000000000..1ecd33f28 --- /dev/null +++ b/results/zed-nix/canary-screening-02/report.md @@ -0,0 +1,36 @@ +# zed-nix: canary-screening-02 + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer + +Observations: 2/4 recorded; 0 failed; 2 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 1 | 2788.14372093361 | 2788.14372093361–2788.14372093361 | unmeasured | 0 | +| Cold build | Cachix | 1 | 2996.7530764946086 | 2996.7530764946086–2996.7530764946086 | unmeasured | 0 | + +Run 37296214039 failed completion checks: + +- Workflow concluded failure +- cachix zed-nix warm: failure +- boringcache zed-nix warm: failure + +Missing observations: + +- Sample 1, BoringCache, Warm build +- Sample 1, Cachix, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 2.3193912506103516 | 2785.8243296829996 | 2788.14372093361 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | Cachix | Cold build | 2.6468822956085205 | 2994.106194199 | 2996.7530764946086 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json b/results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..2b7fd338c --- /dev/null +++ b/results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json @@ -0,0 +1,187 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3193912506103516, + "build_seconds": 2785.8243296829996, + "workflow_seconds": null, + "total_seconds": 2788.14372093361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2788.14372093361 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173730", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] +} diff --git a/results/zed-nix/canary-screening-02/runs/1-cachix-cold.json b/results/zed-nix/canary-screening-02/runs/1-cachix-cold.json new file mode 100644 index 000000000..85046bba8 --- /dev/null +++ b/results/zed-nix/canary-screening-02/runs/1-cachix-cold.json @@ -0,0 +1,145 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.6468822956085205, + "build_seconds": 2994.106194199, + "workflow_seconds": null, + "total_seconds": 2996.7530764946086, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2996.7530764946086 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173731", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] +} diff --git a/schemas/case.schema.json b/schemas/case.schema.json index cb07e5bde..f602792d5 100644 --- a/schemas/case.schema.json +++ b/schemas/case.schema.json @@ -60,7 +60,8 @@ "variants": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}}} } }, - "sync": {"enum": ["upstream-head", "adjacent-pair", "verified-pair", "fixed"]}, + "sync": {"enum": ["upstream-head", "adjacent-pair", "build-relevant-pair", "verified-pair", "fixed"]}, + "rolling_workflow": {"type": "string", "pattern": "^\\.github/workflows/[A-Za-z0-9._-]+\\.ya?ml$"}, "sync_minute": {"type": "integer", "minimum": 1, "maximum": 29}, "source_prefix": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]*$"}, "cache_tags": {"type": "object", "minProperties": 1, "additionalProperties": {"type": "string", "pattern": "^[a-z0-9._-]*\\{scope\\}[a-z0-9._-]*$"}}, diff --git a/scripts/benchmark-cadence.rb b/scripts/benchmark-cadence.rb new file mode 100644 index 000000000..ba56e8615 --- /dev/null +++ b/scripts/benchmark-cadence.rb @@ -0,0 +1,113 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require_relative "benchmark-cases" +require_relative "nightly-canaries" + +module BenchmarkCadence + class Error < StandardError; end + + def self.cases(root: BenchmarkCases::ROOT) + suite = JSON.parse(File.read(File.join(root, "suites/scheduled.json"))) + raise Error, "Unsupported scheduled suite version" unless suite.fetch("schema_version") == 1 + entries = suite.fetch("cases") + raise Error, "Scheduled cases must be unique" unless entries.uniq == entries + entries.each do |entry| + raise Error, "Use case_id and an optional variant" unless (entry.keys - %w[case_id variant]).empty? + BenchmarkCases.load_case(entry.fetch("case_id"), root) + end + entries + end + + def self.fresh_targets(root: BenchmarkCases::ROOT, case_id: nil) + cases(root: root).select { |entry| !case_id || entry.fetch("case_id") == case_id }.flat_map do |entry| + item = BenchmarkCases.load_case(entry.fetch("case_id"), root) + workflows = item.fetch("execution").fetch("workflows").select { |workflow| workflow.fetch("lane") == "fresh" } + raise Error, "#{item.fetch('id')} has no fresh workflow" if workflows.empty? + workflows.map do |workflow| + plan = BenchmarkCases.plan(item, lane: "fresh", workflow: File.basename(workflow.fetch("path")), variant: entry["variant"], root: root) + {"source_repo" => BenchmarkCases::REPOSITORY, "case_id" => item.fetch("id"), + "fresh_workflow" => plan.fetch("workflow"), "fresh_inputs" => plan.fetch("inputs")} + end + end + end + + def self.source_cases(root: BenchmarkCases::ROOT, case_id: nil) + ids = cases(root: root).map { |entry| entry.fetch("case_id") }.uniq + raise Error, "#{case_id} is not in the scheduled suite" if case_id && !ids.include?(case_id) + ids.select { |id| !case_id || id == case_id }.map { |id| BenchmarkCases.load_case(id, root) } + end + + def self.rolling_targets(root: BenchmarkCases::ROOT, case_id:, inputs: {}, item: nil) + source_cases(root: root, case_id: case_id) + item ||= BenchmarkCases.load_case(case_id, root) + workflows = item.fetch("execution").fetch("workflows").select { |workflow| workflow.fetch("lane") == "rolling" } + if (selected = item.dig("execution", "rolling_workflow")) + workflows.select! { |workflow| workflow.fetch("path") == selected } + end + raise Error, "#{case_id} has no rolling workflow" if workflows.empty? + cases(root: root).select { |entry| entry.fetch("case_id") == case_id }.flat_map do |entry| + workflows.map do |workflow| + BenchmarkCases.plan(item, lane: "rolling", workflow: File.basename(workflow.fetch("path")), + variant: entry["variant"], inputs: inputs, root: root) + end + end.uniq + end + + def self.verify_cli(version, runs, probe: method(:cli_help)) + return unless runs.any? { |run| run.fetch("workflow").start_with?("reapi-") } + help = probe.call(version, "cache-registry") + unless help.match?(/--reapi-port\b/) + raise Error, "#{version} does not support cache-registry --reapi-port required by the REAPI cases; no builds dispatched" + end + end + + def self.cli_help(version, command) + require_relative "reapi-setup" + platform = RbConfig::CONFIG.fetch("host_os") + asset = if platform.include?("darwin") + "boringcache-macos-universal" + elsif platform.include?("linux") + "boringcache-linux-#{RbConfig::CONFIG.fetch('host_cpu').match?(/aarch64|arm64/) ? 'arm64' : 'amd64'}" + else + raise Error, "CLI capability inspection requires Linux or macOS" + end + Dir.mktmpdir("benchmark-cli-") do |directory| + binary = File.join(directory, "boringcache") + ReapiSetup.download("boringcache/cli", version, asset, "boringcache", destination: binary) + BenchmarkCases.command(binary, command, "--help") + end + end +end + +if $PROGRAM_NAME == __FILE__ + options = {ref: "main"} + OptionParser.new do |parser| + parser.on("--check") { options[:check] = true } + parser.on("--collect") { options[:collect] = true } + parser.on("--case ID") { |value| options[:case_id] = value } + parser.on("--channel CHANNEL") { |value| options[:channel] = value } + parser.on("--version TAG") { |value| options[:version] = value } + parser.on("--ref REF") { |value| options[:ref] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--summary PATH") { |value| options[:summary] = value } + parser.on("--dry-run") { options[:dry_run] = true } + end.parse! + begin + targets = BenchmarkCadence.fresh_targets(case_id: options.delete(:case_id)) + raise BenchmarkCadence::Error, "No scheduled cases selected" if targets.empty? + if options.delete(:collect) + exit(NightlyCanaries::Runner.new.collect(summary: options.fetch(:summary), benchmarks: targets) ? 0 : 1) + elsif options.delete(:check) + puts "Validated #{targets.length} scheduled fresh workflows" + else + raise BenchmarkCadence::Error, "Use stable or canary" unless %w[stable canary].include?(options[:channel]) + raise BenchmarkCadence::Error, "Inspect the previous receipt before retrying a dispatch" if ENV.fetch("GITHUB_RUN_ATTEMPT", "1").to_i > 1 && !options[:dry_run] + NightlyCanaries::Runner.new.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: targets, + preflight: BenchmarkCadence.method(:verify_cli), **options) + end + rescue BenchmarkCadence::Error, BenchmarkCases::Error, NightlyCanaries::Error, KeyError => error + abort error.message + end +end diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index 098bfe22c..a6aa7a52d 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -273,6 +273,7 @@ def self.update_source(item, workload, root: ROOT) def self.sync_source(item, root: ROOT) kind = item.fetch("execution").fetch("sync", "fixed") raise Error, "#{item.fetch('id')} has fixed source pins" if kind == "fixed" + return sync_snapshot(item, root: root) if kind == "upstream-head" && item.dig("source", "revision") if kind == "upstream-head" pin = item.fetch("source").fetch("pins").find { |value| value["kind"] == "gitlink" } branch, _, status = Open3.capture3("git", "config", "-f", File.join(root, "cases", item.fetch("id"), "payload", ".gitmodules"), "--get", "submodule.#{pin.fetch('path')}.branch") @@ -283,8 +284,13 @@ def self.sync_source(item, root: ROOT) Dir.mktmpdir("benchmark-source-") do |directory| workload = prepare(item, directory: File.join(directory, "workload"), root: root) if kind == "upstream-head" - command("git", "submodule", "update", "--init", "--checkout", "upstream", chdir: workload) source = File.join(workload, "upstream") + # Inspect only the candidate tree. Ancestry is checked against the exact + # declared pin through the compare endpoint below. + unless File.exist?(File.join(source, ".git")) + command("git", "init", source) + command("git", "remote", "add", "origin", "https://github.com/#{item.dig('source', 'repository')}.git", chdir: source) + end command("git", "fetch", "--depth", "1", "origin", "refs/heads/#{branch}", chdir: source) next_sha = command("git", "rev-parse", "FETCH_HEAD", chdir: source).strip current_sha = item.fetch("source").fetch("pins").find { |pin| pin["kind"] == "gitlink" }.fetch("revision") @@ -295,9 +301,10 @@ def self.sync_source(item, root: ROOT) raise Error, "Upstream source is not a fast-forward from the declared pin" unless comparison["status"] == "ahead" && comparison.dig("merge_base_commit", "sha") == current_sha verify_recipe(workload) update_source(item, workload, root: root) - {"case_id" => item.fetch("id"), "updated" => true, "head_sha" => next_sha} + {"case_id" => item.fetch("id"), "updated" => true, "base_sha" => current_sha, "head_sha" => next_sha} else old_source = File.read(File.join(workload, "benchmark-source.env")) + old_settings = BenchmarkPlan.settings(File.join(workload, "benchmark-source.env")) output_path = File.join(directory, "outputs") args = ["bash", "scripts/advance-source-pair.sh", "benchmark-source.env", item.fetch("execution").fetch("source_prefix")] args << "check_dependencies" if kind == "verified-pair" @@ -320,12 +327,49 @@ def self.sync_source(item, root: ROOT) end end update_source(item, workload, root: root) if updated && kind != "verified-pair" + fields.merge!("base_sha" => values.fetch("#{prefix}_BASE_SHA"), "head_sha" => values.fetch("#{prefix}_HEAD_SHA"), + "previous_sha" => old_settings.fetch("#{prefix}_HEAD_SHA")) if updated fields.merge("case_id" => item.fetch("id"), "updated" => updated, "requires_verified_build" => kind == "verified-pair", "candidate_source_env" => updated && File.read(File.join(workload, "benchmark-source.env"))) end end end + def self.sync_snapshot(item, root: ROOT) + repository = item.dig("source", "repository") + current = item.dig("source", "revision") + branch = item.dig("origin", "branch") + head = JSON.parse(command("gh", "api", "repos/#{repository}/commits/#{CGI.escape(branch)}")).fetch("sha") + return {"case_id" => item.fetch("id"), "updated" => false} if head == current + raise Error, "Upstream returned an invalid revision" unless head.match?(/\A[0-9a-f]{40}\z/) + comparison = JSON.parse(command("gh", "api", "repos/#{repository}/compare/#{current}...#{head}")) + unless comparison["status"] == "ahead" && comparison.dig("merge_base_commit", "sha") == current + raise Error, "Upstream source is not a fast-forward from the declared pin" + end + candidate = Marshal.load(Marshal.dump(item)) + candidate.fetch("source")["revision"] = head + candidate.dig("source", "pins").each do |pin| + raise Error, "Snapshot has independently pinned sources; review them before advancement" unless pin["revision"] == current + pin["revision"] = head + end + Dir.mktmpdir("benchmark-source-") do |directory| + workload = prepare(candidate, directory: File.join(directory, "workload"), root: root) + command(RbConfig.ruby, File.join(workload, "scripts/verify-upstream-recipe.rb"), workload, chdir: workload) + end + # Only the exact source URI changes; recipe digests and build flags stay reviewed. + payload = File.join(root, "cases", item.fetch("id"), "payload") + reference = "github:#{repository}/#{current}#default" + replacement = "github:#{repository}/#{head}#default" + %w[.boringcache.toml recipe-contract.json].each do |name| + path = File.join(payload, name) + next unless File.file?(path) + content = File.read(path) + File.write(path, content.gsub(reference, replacement)) if content.include?(reference) + end + write_json(File.join(root, "cases", item.fetch("id"), "case.json"), candidate) + {"case_id" => item.fetch("id"), "updated" => true, "base_sha" => current, "head_sha" => head} + end + def self.verify_recipe(workload) path = File.join(workload, "recipe-contract.json") raise Error, "Source advancement requires a reviewed recipe contract" unless File.file?(path) @@ -537,6 +581,16 @@ def self.write_contract_views(directory, root: ROOT) Dir[File.join(target, ".github", "**", "*.{yml,yaml}")].each do |path| next if path == provider_path document = YAML.safe_load(File.read(path), aliases: true) + if File.basename(path).match?(/\A(?:reapi|nix)-(?:fresh|rolling)-benchmark\.yml\z/) + inputs = (document["on"] || document[true]).dig("workflow_dispatch", "inputs") + inputs.fetch("case_id")["default"] = item.fetch("id") + providers = inputs.fetch("provider").fetch("options") - ["all"] + raise Error, "Provider selector differs from the declared comparison" unless providers.sort == item.dig("comparison", "providers").sort + document.fetch("jobs").each_value do |job| + matrix = job.dig("strategy", "matrix") + matrix["provider"] = providers if matrix && matrix["provider"].is_a?(String) + end + end if item.dig("execution", "native") && File.basename(path).start_with?("native-") recipe = NativeCase.resolve(item.dig("execution", "native"), native_variant) document["env"]["BENCHMARK_ID"] = "${{ format('#{recipe.fetch('benchmark_id')}{0}', inputs.benchmark_id_suffix) }}" diff --git a/scripts/benchmark-phase.rb b/scripts/benchmark-phase.rb index 2e48586b1..a47955316 100644 --- a/scripts/benchmark-phase.rb +++ b/scripts/benchmark-phase.rb @@ -8,6 +8,11 @@ def self.scope(env, require_published: false) lane = env.fetch("CACHE_LANE") raise "Use fresh or rolling" unless %w[fresh rolling].include?(lane) raise "Invalid benchmark ID" unless id.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) + rolling_scope = env.fetch("BENCHMARK_ROLLING_SCOPE", "") + if lane == "rolling" && !rolling_scope.empty? + raise "Invalid rolling cache scope" unless rolling_scope.match?(/\A[a-z0-9][a-z0-9._-]+\z/) + return "#{id}-rolling-#{rolling_scope}" + end series = env.fetch("BENCHMARK_SERIES_ID", "") if !series.empty? raise "Invalid series ID" unless series.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) diff --git a/scripts/cadence-dispatch.rb b/scripts/cadence-dispatch.rb new file mode 100644 index 000000000..30422297d --- /dev/null +++ b/scripts/cadence-dispatch.rb @@ -0,0 +1,107 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require_relative "benchmark-cadence" + +# The parent validates the entire suite and resolves one release before the +# matrix starts. Each child records its request independently, including errors. +module CadenceDispatch + class Error < StandardError; end + + def self.matrix(plan) + raise Error, "Expected a validated dispatch plan" unless plan.fetch("state") == "planned" + {"include" => plan.fetch("runs").each_with_index.map do |run, index| + {"index" => index, "label" => NightlyCanaries::Runner.new.workload_name(run)} + end} + end + + def self.dispatch(plan, index:, output:, runner: NightlyCanaries::Runner.new) + raise Error, "Inspect the previous receipt before retrying a dispatch" if ENV.fetch("GITHUB_RUN_ATTEMPT", "1").to_i > 1 + matrix(plan) + raise Error, "Invalid dispatch target index" unless index.is_a?(Integer) && index >= 0 && index < plan.fetch("runs").length + run = plan.fetch("runs").fetch(index).dup + record = plan.except("runs").merge("runs" => [run], "target_index" => index, "state" => "dispatching") + # A lost HTTP response cannot establish whether GitHub accepted the request. + run["state"] = "requesting" + runner.write(output, record) + result = runner.api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}/dispatches", + body: {"ref" => plan.fetch("ref"), "inputs" => run.fetch("inputs").merge("cli_version" => plan.fetch("cli_version")), "return_run_details" => true}) + id = result["workflow_run_id"] + raise Error, "Dispatch returned no run ID; inspect the retained request before retrying" unless id.is_a?(Integer) && id.positive? + run.merge!("id" => id, "url" => "https://github.com/#{run.fetch('repository')}/actions/runs/#{id}", "state" => "requested") + record["state"] = "requested" + runner.write(output, record) + record + rescue StandardError => error + if record + run["state"] = "request-unknown" unless run["id"] + run["error"] = error.message + record.merge!("state" => "dispatch-failed", "error" => error.message) + runner.write(output, record) + end + raise + end + + def self.combine(plan, receipts, dry_run: false) + matrix(plan) + result = Marshal.load(Marshal.dump(plan)) + seen = [] + receipts.each do |receipt| + index = receipt.fetch("target_index") + raise Error, "Duplicate or invalid dispatch receipt" unless index.is_a?(Integer) && index >= 0 && index < result.fetch("runs").length && !seen.include?(index) + seen << index + %w[cli_version channel ref].each do |key| + raise Error, "Dispatch receipt #{key} differs from the plan" unless receipt.fetch(key) == plan.fetch(key) + end + raise Error, "Expected one target per receipt" unless receipt.fetch("runs").length == 1 + run = receipt.fetch("runs").first + expected = plan.fetch("runs").fetch(index) + %w[repository workflow inputs].each do |key| + raise Error, "Dispatch receipt target differs from the plan" unless run.fetch(key) == expected.fetch(key) + end + if run["state"] == "requested" + raise Error, "Requested run lacks its GitHub ID" unless run["id"].is_a?(Integer) && run["id"].positive? + end + result.fetch("runs")[index] = run + end + result["state"] = if dry_run && receipts.empty? + "planned" + elsif result.fetch("runs").all? { |run| run["state"] == "requested" } + "requested" + else + "dispatch-failed" + end + result + end +end + +if $PROGRAM_NAME == __FILE__ + options = {} + OptionParser.new do |parser| + parser.on("--plan PATH") { |value| options[:plan] = value } + parser.on("--matrix") { options[:matrix] = true } + parser.on("--index N", Integer) { |value| options[:index] = value } + parser.on("--receipts PATH") { |value| options[:receipts] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--summary PATH") { |value| options[:summary] = value } + parser.on("--dry-run") { options[:dry_run] = true } + end.parse! + begin + plan = JSON.parse(File.read(options.fetch(:plan))) + if options[:matrix] + puts JSON.generate(CadenceDispatch.matrix(plan)) + elsif options.key?(:index) + CadenceDispatch.dispatch(plan, index: options.fetch(:index), output: options.fetch(:output)) + else + receipts = Dir.glob(File.join(options.fetch(:receipts), "**/target-*.json")).sort.map { |path| JSON.parse(File.read(path)) } + record = CadenceDispatch.combine(plan, receipts, dry_run: options[:dry_run]) + runner = NightlyCanaries::Runner.new + runner.write(options.fetch(:output), record) + runner.write_summary(options[:summary], record) + exit(record.fetch("state") == "dispatch-failed" ? 1 : 0) + end + rescue CadenceDispatch::Error, NightlyCanaries::Error, KeyError, JSON::ParserError => error + abort error.message + end +end diff --git a/scripts/check-registry-alignment.rb b/scripts/check-registry-alignment.rb index bfda7a68e..c4beb9bdd 100755 --- a/scripts/check-registry-alignment.rb +++ b/scripts/check-registry-alignment.rb @@ -11,7 +11,8 @@ errors = [] suites.each do |name, suite| next unless suite.is_a?(Hash) - Array(suite["cases"]).each do |id| + Array(suite["cases"]).each do |entry| + id = entry.is_a?(Hash) ? entry.fetch("case_id") : entry item = by_id[id] errors << "#{name}: unknown case #{id}" unless item errors << "#{name}: retained case #{id} cannot execute" if item && item["kind"] == "retained" diff --git a/scripts/check-report-contract.rb b/scripts/check-report-contract.rb index 703c4f296..92b5c258e 100755 --- a/scripts/check-report-contract.rb +++ b/scripts/check-report-contract.rb @@ -209,7 +209,13 @@ def producers_for(workflow:, repo_dir:, job_id:, job:) next unless step.is_a?(Hash) uses = step["uses"].to_s - if uses.start_with?("./.github/actions/") + if %w[./.github/actions/reapi-benchmark ./.github/actions/nix-benchmark].include?(uses) + with = step.fetch("with") + fields = {"benchmark" => workflow.resolve(with.fetch("case-id"), matrix), + "strategy" => workflow.resolve(with.fetch("provider"), matrix), + "lane" => workflow.resolve(with.fetch("lane", "fresh"), matrix), + "phase" => workflow.resolve(with.fetch("phase"), matrix), "variant" => ""} + elsif uses.start_with?("./.github/actions/") with = step["with"].is_a?(Hash) ? step["with"] : {} variant_input = action_variant_input(repo_dir, uses, with) variant = workflow.resolve(with[variant_input].to_s, matrix).to_s if variant_input @@ -220,6 +226,12 @@ def producers_for(workflow:, repo_dir:, job_id:, job:) "phase" => workflow.resolve(with["phase"].to_s, matrix), "variant" => variant_input && variant != DEFAULT_PLATFORM ? variant : "" } + elsif step["run"].to_s.include?("write_phase_result.rb") + flags = step.fetch("run").scan(/--(surface|strategy|phase)\s+([^\s\\]+)/).to_h + surface = flags.fetch("surface") + base = flags.fetch("phase") == "base" + fields = {"benchmark" => step.fetch("run").include?("--continuous") ? "obs-studio" : "obs-studio-#{surface}", + "strategy" => flags.fetch("strategy"), "lane" => base ? "fresh" : "rolling", "phase" => base ? "cold" : "commit", "variant" => surface} elsif step["run"].to_s.match?(/benchmark-report\.rb\s+phase\b/) shell_env = workflow.shell_env_for(job, step, matrix) fields = phase_flags(step["run"].to_s).transform_values { |value| workflow.resolve(value, matrix, shell_env) } diff --git a/scripts/nightly-canaries.rb b/scripts/nightly-canaries.rb index 3cd2f8025..c58c54853 100755 --- a/scripts/nightly-canaries.rb +++ b/scripts/nightly-canaries.rb @@ -30,9 +30,24 @@ def published_canary(release) unless release["prerelease"] && !release["draft"] && release.fetch("tag_name").match?(/\Avcli-canary-[0-9a-f]{9,40}\z/) raise Error, "Use a published CLI canary" end + published_assets(release) + end + + def latest_stable + published_stable(api("repos/boringcache/cli/releases/latest")) + end + + def published_stable(release) + unless !release["prerelease"] && !release["draft"] && release.fetch("tag_name").match?(/\Av\d+\.\d+\.\d+\z/) + raise Error, "Use a published stable CLI release" + end + published_assets(release) + end + + def published_assets(release) names = release.fetch("assets").map { |asset| asset.fetch("name") } required = %w[SHA256SUMS boringcache-linux-amd64 boringcache-linux-arm64 boringcache-macos-universal boringcache-windows-amd64.exe] - raise Error, "The CLI canary is missing release assets" unless (required - names).empty? + raise Error, "The CLI release is missing release assets" unless (required - names).empty? release.fetch("tag_name") end @@ -52,17 +67,20 @@ def targets(benchmarks) end.uniq end - def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, benchmarks: BENCHMARKS) + def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, benchmarks: BENCHMARKS, channel: "canary", ref: "main", preflight: nil) + raise Error, "Use stable or canary" unless %w[stable canary].include?(channel) selected = targets(benchmarks).select { |repo, _| repo == repository } raise Error, "No active fresh workflows registered for #{repository}" if selected.empty? if version.nil? || version.empty? - version = latest_canary + version = channel == "canary" ? latest_canary : latest_stable else - raise Error, "Use an exact CLI canary tag" unless version.match?(/\Avcli-canary-[0-9a-f]{9,40}\z/) - published_canary(api("repos/boringcache/cli/releases/tags/#{version}")) + pattern = channel == "canary" ? /\Avcli-canary-[0-9a-f]{9,40}\z/ : /\Av\d+\.\d+\.\d+\z/ + raise Error, "Use an exact CLI #{channel} tag" unless version.match?(pattern) + release = api("repos/boringcache/cli/releases/tags/#{version}") + channel == "canary" ? published_canary(release) : published_stable(release) end record = {"cli_version" => version, "created_at" => Time.now.utc.iso8601, - "state" => "dispatching", "runs" => []} + "channel" => channel, "ref" => ref, "state" => "dispatching", "runs" => []} selected.each do |repository, workflow, inputs| raise Error, "Benchmark lacks a fresh workflow" if repository.nil? || workflow.nil? record["runs"] << {"repository" => repository, "workflow" => workflow, "inputs" => inputs, "state" => "planned"} @@ -73,10 +91,11 @@ def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, b workflow = api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}") raise Error, "#{run.fetch('repository')} workflow is not active" unless workflow["state"] == "active" end + preflight.call(version, record.fetch("runs")) if preflight record["runs"].each do |run| next if dry_run result = api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}/dispatches", - body: {"ref" => "main", "inputs" => run.fetch("inputs").merge("cli_version" => version), "return_run_details" => true}) + body: {"ref" => ref, "inputs" => run.fetch("inputs").merge("cli_version" => version), "return_run_details" => true}) id = result["workflow_run_id"] raise Error, "Dispatch returned no run ID; inspect the workflow before retrying" unless id.is_a?(Integer) && id.positive? run.merge!("id" => id, "url" => "https://github.com/#{run.fetch('repository')}/actions/runs/#{id}", "state" => "requested") @@ -111,10 +130,11 @@ def check(record, summary:) def receipt(repository, run_id) Dir.mktmpdir("canary-receipt-") do |directory| + artifact = repository == "boringcache/benchmarks" ? "benchmark-dispatch" : "nightly-canaries" _, error, status = Open3.capture3("gh", "run", "download", run_id.to_s, - "--repo", repository, "--name", "nightly-canaries", "--dir", directory) + "--repo", repository, "--name", artifact, "--dir", directory) raise Error, "Cannot read dispatch receipt: #{error.strip}" unless status.success? - JSON.parse(File.read(File.join(directory, "nightly-canaries.json"))) + JSON.parse(File.read(File.join(directory, "#{artifact}.json"))) end end @@ -184,7 +204,7 @@ def write(path, record) def write_summary(path, record) return unless path - lines = ["## CLI canary benchmarks", "", "CLI: `#{record.fetch('cli_version')}`", "", + lines = ["## Fresh benchmark runs", "", "CLI: `#{record.fetch('cli_version')}`", "", "Dispatch: **#{record.fetch('state')}**. Benchmark outcomes are listed separately.", ""] record.fetch("runs").each do |run| name = "#{run.fetch('repository')} / #{workload_name(run)}" diff --git a/scripts/nix-benchmark.rb b/scripts/nix-benchmark.rb index 8dfaa0bca..d22ddfea1 100644 --- a/scripts/nix-benchmark.rb +++ b/scripts/nix-benchmark.rb @@ -4,6 +4,7 @@ require "open3" require "fileutils" require "uri" +require "digest" require_relative "benchmark-plan" module NixBenchmark @@ -35,7 +36,7 @@ def self.closure(paths) end.to_h end - def self.prepare + def self.prepare(seed: nil) recipe = BenchmarkPlan.load.fetch("adapters").fetch("nix") command = recipe.fetch("command") reference = command.fetch(2) @@ -48,7 +49,34 @@ def self.prepare realized = capture("nix-store", "--realise", *dependencies).lines.map(&:strip) outputs = capture("nix-store", "--query", "--outputs", drv).lines.map(&:strip) raise Error, "Package output already exists before measurement" if outputs.any? { |path| File.exist?(path) } - write("nix-evidence/baseline.json", {"derivation" => drv, "outputs" => outputs, "dependencies" => closure(realized), "nix_version" => capture("nix", "--version")}) + baseline = {"derivation" => drv, "outputs" => outputs, "dependencies" => closure(realized), "nix_version" => capture("nix", "--version")} + if seed + expected = JSON.parse(File.read(File.join(seed, "baseline.json"))) + raise Error, "Dependency seed differs from the declared workload" unless baseline == expected + end + write("nix-evidence/baseline.json", baseline) + end + + def self.export_dependencies(directory: "nix-dependencies") + baseline = JSON.parse(File.read("nix-evidence/baseline.json")) + paths = baseline.fetch("dependencies").keys + raise Error, "Dependency export includes the measured package" unless (paths & baseline.fetch("outputs")).empty? + raise Error, "No dependency store paths to export" if paths.empty? + FileUtils.mkdir_p(directory) + archive = File.join(directory, "store.nar.zst") + statuses = Open3.pipeline(["nix-store", "--export", *paths], ["zstd", "-T0", "-3", "-o", archive]) + raise Error, "Dependency export failed" unless statuses.all?(&:success?) + write(File.join(directory, "baseline.json"), baseline) + write(File.join(directory, "archive.json"), {"sha256" => Digest::SHA256.file(archive).hexdigest}) + end + + def self.import_dependencies(directory:) + archive = File.join(directory, "store.nar.zst") + expected = JSON.parse(File.read(File.join(directory, "archive.json"))).fetch("sha256") + raise Error, "Dependency archive checksum differs" unless Digest::SHA256.file(archive).hexdigest == expected + statuses = Open3.pipeline(["zstd", "-dc", archive], ["nix-store", "--import"]) + raise Error, "Dependency import failed" unless statuses.all?(&:success?) + prepare(seed: directory) end def self.provider_substituter(config, provider, cache_name) @@ -68,18 +96,24 @@ def self.provider_substituter(config, provider, cache_name) matches.first end - def self.build(phase:, provider:, cache_name: "") - raise Error, "Use cold or warm" unless %w[cold warm].include?(phase) - baseline = JSON.parse(File.read("nix-evidence/baseline.json")) - raise Error, "Package output already exists before measurement" if baseline.fetch("outputs").any? { |path| File.exist?(path) } - command = BenchmarkPlan.command("nix") + def self.build_command(phase:, provider:, cache_name: "") + raise Error, "Use cold, warm or commit" unless %w[cold warm commit].include?(phase) + command = BenchmarkPlan.command("nix").dup if phase == "cold" command += %w[--option substitute false] else config = JSON.parse(capture("nix", "config", "show", "--json")) url = provider_substituter(config, provider, cache_name) - command += ["--max-jobs", "0", "--builders", "", "--option", "substituters", url, "--option", "extra-substituters", ""] + command += ["--option", "substituters", url, "--option", "extra-substituters", ""] + command += ["--max-jobs", "0", "--builders", ""] if phase == "warm" end + command + end + + def self.build(phase:, provider:, cache_name: "") + baseline = JSON.parse(File.read("nix-evidence/baseline.json")) + raise Error, "Package output already exists before measurement" if baseline.fetch("outputs").any? { |path| File.exist?(path) } + command = build_command(phase: phase, provider: provider, cache_name: cache_name) started = Process.clock_gettime(Process::CLOCK_MONOTONIC) success = false File.open("nix-evidence/build.log", "w") do |log| @@ -125,6 +159,8 @@ def self.compare(seed:, current: "nix-evidence") begin case ARGV.shift when "prepare" then NixBenchmark.prepare + when "export-dependencies" then NixBenchmark.export_dependencies + when "import-dependencies" then NixBenchmark.import_dependencies(directory: ARGV.fetch(0)) when "build" then NixBenchmark.build(phase: ENV.fetch("NIX_PHASE"), provider: ENV.fetch("NIX_PROVIDER"), cache_name: ENV.fetch("CACHIX_CACHE", "")) when "verify" then NixBenchmark.verify(phase: ENV.fetch("NIX_PHASE"), provider: ENV.fetch("NIX_PROVIDER"), cache_name: ENV.fetch("CACHIX_CACHE", "")) when "compare" then NixBenchmark.compare(seed: ARGV.fetch(0)) diff --git a/scripts/reapi-registry.rb b/scripts/reapi-registry.rb index 88b460b4f..2a8a4230c 100644 --- a/scripts/reapi-registry.rb +++ b/scripts/reapi-registry.rb @@ -60,7 +60,7 @@ def self.with_process(command, port:, log:, startup_timeout: 120, shutdown_timeo end def self.command(provider:, phase:, workspace:, tag:, port: 5060) - raise Error, "Use cold or warm" unless %w[cold warm].include?(phase) + raise Error, "Use cold, warm or commit" unless %w[cold warm commit].include?(phase) case provider when "boringcache" args = ["boringcache", "cache-registry", workspace, tag, "--port", "5061", "--reapi-port", port.to_s, diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index 27393d5b1..86d65b38d 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -12,7 +12,7 @@ def self.run(*command) raise "Command failed: #{command.first}" unless system(*command) end - def self.download(repository, release, asset, binary, sha256: nil) + def self.download(repository, release, asset, binary, sha256: nil, destination: nil) Dir.mktmpdir("reapi-download-") do |directory| run("gh", "release", "download", release, "--repo", repository, "--pattern", asset, "--dir", directory) path = File.join(directory, asset) @@ -22,7 +22,7 @@ def self.download(repository, release, asset, binary, sha256: nil) sha256 = rows.find { |row| row[1] == asset }&.first end raise "Release checksum is missing or different: #{asset}" unless sha256&.match?(/\A[0-9a-f]{64}\z/) && Digest::SHA256.file(path).hexdigest == sha256 - destination = File.join(ENV.fetch("RUNNER_TEMP"), "reapi-bin", binary) + destination ||= File.join(ENV.fetch("RUNNER_TEMP"), "reapi-bin", binary) FileUtils.mkdir_p(File.dirname(destination)) if asset.end_with?(".zst") run("zstd", "-d", path, "-o", destination) @@ -61,12 +61,31 @@ def self.prepare ReapiClient.recipe.fetch("prepare").each { |command| run(*command) } end + def self.rolling_seed + context = JSON.parse(File.read("benchmark-context.json")) + path = "reapi-store/benchmark-source.json" + previous = File.file?(path) ? JSON.parse(File.read(path)) : nil + if previous && previous.fetch("case_id") != ENV.fetch("BENCHMARK_ID") + raise "Rolling comparator seed belongs to another case" + end + seed = {"cache_key" => ENV.fetch("RESTORED_CACHE_KEY", ""), "previous" => previous, + "current" => {"case_id" => ENV.fetch("BENCHMARK_ID"), "source" => context.fetch("source"), + "run_id" => ENV.fetch("GITHUB_RUN_ID"), "run_attempt" => ENV.fetch("GITHUB_RUN_ATTEMPT")}} + File.write("reapi-evidence/rolling-seed.json", JSON.pretty_generate(seed) + "\n") + end + + def self.publish_seed + seed = JSON.parse(File.read("reapi-evidence/rolling-seed.json")) + FileUtils.mkdir_p("reapi-store") + File.write("reapi-store/benchmark-source.json", JSON.pretty_generate(seed.fetch("current")) + "\n") + end + def self.report require_relative "benchmark-report" context = JSON.parse(File.read("benchmark-context.json")) timings = JSON.parse(File.read("reapi-evidence/timing.json")) args = ["phase", "--verified-output", "--benchmark", ENV.fetch("BENCHMARK_ID"), "--strategy", ENV.fetch("PROVIDER"), - "--lane", "fresh", "--phase", ENV.fetch("PHASE"), "--mode", "reapi", + "--lane", ENV.fetch("CACHE_LANE", "fresh"), "--phase", ENV.fetch("PHASE"), "--mode", "reapi", "--workspace", "boringcache/benchmarks", "--cache-tag", BenchmarkPhase.scope(ENV), "--source-repository", context.dig("source", "repository"), "--source-sha", context.dig("source", "revision"), "--build-seconds", timings.fetch("build_seconds").to_s, "--restore-or-setup-seconds", timings.fetch("restore_or_setup_seconds").to_s, @@ -87,6 +106,8 @@ def self.report when "install" then ReapiSetup.install when "prepare" then ReapiSetup.prepare when "report" then ReapiSetup.report + when "rolling-seed" then ReapiSetup.rolling_seed + when "publish-seed" then ReapiSetup.publish_seed else abort "Use install, prepare or report" end end diff --git a/scripts/source-promotion.rb b/scripts/source-promotion.rb new file mode 100644 index 000000000..1c5eb0b01 --- /dev/null +++ b/scripts/source-promotion.rb @@ -0,0 +1,291 @@ +# frozen_string_literal: true + +require "base64" +require_relative "benchmark-cadence" + +# Publication uses reviewed harness code only. Candidate workloads run in jobs +# with read-only repository access; this module never executes their code. +module SourcePromotion + class Error < StandardError; end + + def self.changes(item, proposal, root: BenchmarkCases::ROOT) + raise Error, "Expected a changed source proposal for this case" unless proposal["state"] == "proposal" && proposal["case_id"] == item.fetch("id") + if !!proposal["requires_verified_build"] != (item.dig("execution", "sync") == "verified-pair") + raise Error, "Source proposal changes the case's build-before-promotion requirement" + end + base, head = proposal.values_at("base_sha", "head_sha") + raise Error, "Source promotion requires distinct exact revisions" unless [base, head].all? { |sha| sha.is_a?(String) && sha.match?(/\A[0-9a-f]{40}\z/) } && base != head + candidate = Marshal.load(Marshal.dump(item)) + prefix = item.dig("execution", "source_prefix") + id = item.fetch("id") + paths = {} + if prefix + path = "cases/#{id}/payload/benchmark-source.env" + original = File.read(File.join(root, path)) + settings = BenchmarkPlan.settings(File.join(root, path)) + previous = proposal.fetch("previous_sha", base) + raise Error, "Candidate does not follow the declared parent" unless settings.fetch("#{prefix}_HEAD_SHA") == previous + raise Error, "An adjacent sequence must retain the previous head as its base" if item.dig("execution", "sync") == "adjacent-pair" && base != previous + expected = original.sub(/^#{prefix}_BASE_SHA=.*$/, "#{prefix}_BASE_SHA=#{base}").sub(/^#{prefix}_HEAD_SHA=.*$/, "#{prefix}_HEAD_SHA=#{head}") + raise Error, "Source proposal changes settings other than the source pair" unless proposal.fetch("candidate_source_env") == expected + paths[path] = expected + candidate.fetch("source")["pins"] = candidate.dig("source", "pins").reject { |pin| pin["kind"] == "env" && pin["path"] == "payload/benchmark-source.env" } + + [base, head].map { |sha| {"path" => "payload/benchmark-source.env", "revision" => sha, "kind" => "env"} } + candidate.dig("source", "pins").select { |pin| pin["kind"] == "gitlink" }.each { |pin| pin["revision"] = head } + elsif item.dig("source", "revision") + raise Error, "Candidate does not follow the declared snapshot" unless item.dig("source", "revision") == base + candidate.fetch("source")["revision"] = head + candidate.dig("source", "pins").each do |pin| + raise Error, "Snapshot contains an independently pinned source" unless pin.fetch("revision") == base + pin["revision"] = head + end + reference = "github:#{item.dig('source', 'repository')}/#{base}#default" + replacement = "github:#{item.dig('source', 'repository')}/#{head}#default" + %w[.boringcache.toml recipe-contract.json].each do |name| + path = "cases/#{id}/payload/#{name}" + next unless File.file?(File.join(root, path)) + original = File.read(File.join(root, path)) + paths[path] = original.gsub(reference, replacement) if original.include?(reference) + end + else + pins = candidate.dig("source", "pins").select { |pin| pin["kind"] == "gitlink" } + raise Error, "Candidate does not follow the declared gitlink" if pins.empty? || pins.any? { |pin| pin.fetch("revision") != base } + pins.each { |pin| pin["revision"] = head } + end + paths["cases/#{id}/case.json"] = JSON.pretty_generate(candidate) + "\n" + [candidate, paths] + end + + def self.targets(item, proposal, version:, root: BenchmarkCases::ROOT) + candidate, = changes(item, proposal, root: root) + inputs = {"cli_version" => version} + case item.dig("execution", "sync") + when "adjacent-pair", "build-relevant-pair" + inputs.merge!("base_sha" => proposal.fetch("base_sha"), "head_sha" => proposal.fetch("head_sha"), "cache_scope" => "rolling-main") + when "verified-pair" + inputs.merge!("base_sha" => proposal.fetch("base_sha"), "head_sha" => proposal.fetch("head_sha"), "source_distance" => proposal.fetch("source_distance").to_s) + end + BenchmarkCadence.rolling_targets(case_id: item.fetch("id"), item: candidate, inputs: inputs, root: root) + end + + class Publisher + attr_reader :root, :repository, :branch + + def initialize(root: BenchmarkCases::ROOT, runner: NightlyCanaries::Runner.new, branch: "main") + raise Error, "Use main or an isolated cadence rehearsal branch" unless branch == "main" || branch.match?(/\Acadence-rehearsal-[a-z0-9-]+\z/) + @root, @runner = root, runner + @branch = branch + @repository = BenchmarkCases::REPOSITORY + end + + def api(path, body: nil) + if path == "graphql" + output, error, status = Open3.capture3("gh", "api", "graphql", "--method", "POST", "--input", "-", stdin_data: JSON.generate(body)) + result = JSON.parse(output) unless output.empty? + return result if result && (status.success? || result.key?("errors")) + raise Error, "GitHub publication request failed: #{error.strip}" + end + @runner.api(path, body: body) + end + + def file(path, ref: branch) + result = api("repos/#{repository}/contents/#{path}?ref=#{CGI.escape(ref)}") + raise Error, "Expected a repository file" unless result["type"] == "file" + Base64.decode64(result.fetch("content")) + end + + def optional_file(path, ref: branch) + file(path, ref: ref) + rescue NightlyCanaries::Error => error + raise unless error.message.include?("HTTP 404") + nil + end + + # Every update is conditional on both the branch head and the previous + # contents of the files this case owns. Another case may advance meanwhile. + def commit(changes, expected:, message:) + 20.times do + head = api("repos/#{repository}/git/ref/heads/#{branch}").dig("object", "sha") + protect_harness(head, except: expected.keys) if @case_id + expected.each do |path, contents| + raise Error, "#{path} changed while this source was being checked" unless optional_file(path, ref: head) == contents + end + input = {"branch" => {"repositoryNameWithOwner" => repository, "branchName" => branch}, + "expectedHeadOid" => head, "message" => {"headline" => message}, + "fileChanges" => {"additions" => changes.map { |path, contents| {"path" => path, "contents" => Base64.strict_encode64(contents)} }}} + result = api("graphql", body: {"query" => 'mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid } } }', "variables" => {"input" => input}}) + oid = result.dig("data", "createCommitOnBranch", "commit", "oid") + if oid&.match?(/\A[0-9a-f]{40}\z/) + @baseline_ref = oid + return oid + end + errors = result.fetch("errors", []) + # Only an optimistic-concurrency conflict is safe to retry. Other API + # failures may have an uncertain outcome and need their receipt inspected. + raise Error, "Source publication failed: #{errors.map { |error| error['message'] }.join('; ')}" unless errors.any? { |error| error["type"] == "STALE_DATA" } + end + raise Error, "Source publication could not acquire the current branch head" + end + + def protect_harness(head, except:) + @baseline_ref ||= BenchmarkCases.command("git", "rev-parse", "HEAD", chdir: root).strip + snapshots = [@baseline_ref, head].map do |ref| + tree = api("repos/#{repository}/git/trees/#{ref}?recursive=1") + raise Error, "Cannot verify a truncated harness tree" if tree["truncated"] + tree.fetch("tree").select do |entry| + path = entry.fetch("path") + entry["type"] == "blob" && !except.include?(path) && + (path.start_with?(".github/", "scripts/", "bin/", "cases/#{@case_id}/") || %w[Gemfile Gemfile.lock .tool-versions suites/scheduled.json].include?(path)) + end.to_h { |entry| [entry.fetch("path"), entry.fetch("sha")] } + end + raise Error, "The case or shared harness changed during source inspection" unless snapshots.first == snapshots.last + end + + def state_path(id) + "migration/rolling/#{id}.json" + end + + def save(record, expected:, changes: {}) + contents = JSON.pretty_generate(record) + "\n" + path = state_path(record.fetch("case_id")) + history = "migration/rolling/#{record.fetch('case_id')}/#{record.fetch('proposal').fetch('head_sha')}.json" + commit(changes.merge(path => contents, history => contents), expected: expected, + message: "Record #{record.fetch('case_id')} rolling #{record.fetch('state')}") + contents + end + + def reconcile(item, output:) + @case_id = item.fetch("id") + path = state_path(item.fetch("id")) + previous = optional_file(path) + return nil unless previous + record = JSON.parse(previous) + raise Error, "Rolling receipt belongs to another case" unless record.fetch("case_id") == item.fetch("id") + if %w[dispatching dispatch-failed].include?(record.fetch("state")) + BenchmarkCases.write_json(output, record) + raise Error, "Inspect the retained dispatch before advancing this case; a request may have been accepted" + end + return record unless record["state"] == "requested" + if record.fetch("runs").empty? || record.fetch("runs").any? { |run| !run["id"].is_a?(Integer) || !run["id"].positive? } + raise Error, "A requested rolling receipt must retain every GitHub run ID" + end + record.fetch("runs").each do |run| + current = api("repos/#{repository}/actions/runs/#{run.fetch('id')}") + run["outcome"] = current["status"] == "completed" ? current.fetch("conclusion") : current.fetch("status") + end + if record.fetch("runs").any? { |run| %w[queued requested pending waiting in_progress].include?(run["outcome"]) } + BenchmarkCases.write_json(output, record) + return record + end + success = record.fetch("runs").all? { |run| run["outcome"] == "success" } + record["state"] = success ? "completed" : "failed" + changes = {} + expected = {path => previous} + if success && record.fetch("proposal")["requires_verified_build"] + _, changes = SourcePromotion.changes(item, record.fetch("proposal"), root: root) + changes.each_key { |name| expected[name] = File.read(File.join(root, name)) } + record["state"] = "promoted" + end + save(record, expected: expected, changes: changes) + BenchmarkCases.write_json(output, record) + record + end + + def publish(item, proposal, version:, output:, dry_run: false) + @case_id = item.fetch("id") + candidate, changes = SourcePromotion.changes(item, proposal, root: root) + plans = SourcePromotion.targets(item, proposal, version: version, root: root) + BenchmarkCadence.verify_cli(version, plans) + path = state_path(item.fetch("id")) + previous = optional_file(path) + if previous + prior = JSON.parse(previous) + raise Error, "This case has an unfinished or uncertain rolling request" if %w[dispatching dispatch-failed requested].include?(prior.fetch("state")) + raise Error, "This source already has a retained rolling observation" if prior.dig("proposal", "head_sha") == proposal.fetch("head_sha") + end + runs = plans.map { |plan| plan.slice("repository", "workflow", "inputs").merge("state" => "requesting") } + record = {"schema_version" => 1, "case_id" => item.fetch("id"), "state" => "dispatching", "cli_version" => version, "ref" => branch, + "proposal" => proposal, "created_at" => Time.now.utc.iso8601, "runs" => runs} + if dry_run + record["state"] = "planned" + runs.each { |run| run["state"] = "planned" } + BenchmarkCases.write_json(output, record) + return record + end + # Zed advances only after the build succeeds. The other controllers retain + # their historical promote-then-build ordering. + changes = {} if proposal["requires_verified_build"] + expected = {path => previous, "cases/#{item.fetch('id')}/case.json" => File.read(File.join(root, "cases", item.fetch("id"), "case.json"))} + changes.each_key { |name| expected[name] = File.read(File.join(root, name)) } + BenchmarkCases.write_json(output, record) + pending = save(record, expected: expected, changes: changes) + runs.each do |run| + begin + result = api("repos/#{repository}/actions/workflows/#{run.fetch('workflow')}/dispatches", + body: {"ref" => branch, "inputs" => run.fetch("inputs"), "return_run_details" => true}) + id = result["workflow_run_id"] + raise Error, "Dispatch returned no run ID" unless id.is_a?(Integer) && id.positive? + run.merge!("id" => id, "url" => "https://github.com/#{repository}/actions/runs/#{id}", "state" => "requested") + rescue StandardError => error + run.merge!("state" => "request-unknown", "error" => error.message) + ensure + BenchmarkCases.write_json(output, record) + end + end + record["state"] = runs.all? { |run| run["state"] == "requested" } ? "requested" : "dispatch-failed" + save(record, expected: {path => pending}) + BenchmarkCases.write_json(output, record) + raise Error, "At least one rolling request has an uncertain outcome; inspect its receipt before retrying" unless record["state"] == "requested" + record + end + end +end + +if $PROGRAM_NAME == __FILE__ + options = {channel: "stable"} + OptionParser.new do |parser| + parser.on("--case ID") { |value| options[:case_id] = value } + parser.on("--proposal PATH") { |value| options[:proposal] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--channel CHANNEL") { |value| options[:channel] = value } + parser.on("--version TAG") { |value| options[:version] = value } + parser.on("--publish") { options[:publish] = true } + parser.on("--reconcile") { options[:reconcile] = true } + end.parse! + begin + item = BenchmarkCadence.source_cases(case_id: options.fetch(:case_id)).fetch(0) + if options[:publish] || options[:reconcile] + unless ENV["BENCHMARK_CADENCE_ACTIVE"] == "true" && ENV["GITHUB_REF_NAME"] == "main" && ENV["GITHUB_REPOSITORY"] == BenchmarkCases::REPOSITORY + raise SourcePromotion::Error, "Source publication requires active central ownership on main" + end + end + publisher = SourcePromotion::Publisher.new + output = options.fetch(:output) + if options[:reconcile] + record = publisher.reconcile(item, output: output) + proceed = !record || !%w[requested dispatching dispatch-failed].include?(record.fetch("state")) + if ENV["GITHUB_OUTPUT"] + head = publisher.api("repos/#{BenchmarkCases::REPOSITORY}/git/ref/heads/main").dig("object", "sha") + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |file| file.puts("proceed=#{proceed}\nref=#{head}") } + end + else + inventory = JSON.parse(File.read(options.fetch(:proposal))) + proposal = inventory.fetch("records").find { |record| record.fetch("case_id") == item.fetch("id") } + raise SourcePromotion::Error, "No changed source proposal" unless proposal && proposal["state"] == "proposal" + runner = NightlyCanaries::Runner.new + raise SourcePromotion::Error, "Use stable or canary" unless %w[stable canary].include?(options[:channel]) + version = if options[:version] + release = runner.api("repos/boringcache/cli/releases/tags/#{options.fetch(:version)}") + options[:channel] == "stable" ? runner.published_stable(release) : runner.published_canary(release) + else + options[:channel] == "stable" ? runner.latest_stable : runner.latest_canary + end + publisher.publish(item, proposal, version: version, output: output, dry_run: !options[:publish]) + end + rescue SourcePromotion::Error, BenchmarkCases::Error, BenchmarkCadence::Error, NightlyCanaries::Error, KeyError, JSON::ParserError => error + if options[:output] && !File.exist?(options[:output]) + BenchmarkCases.write_json(options[:output], {"case_id" => options[:case_id], "state" => "blocked", "error" => error.message}) + end + abort error.message + end +end diff --git a/scripts/sync-sources.rb b/scripts/sync-sources.rb index 2521e80cc..2345e1633 100755 --- a/scripts/sync-sources.rb +++ b/scripts/sync-sources.rb @@ -4,23 +4,45 @@ require "bundler/setup" require "optparse" require_relative "benchmark-cases" +require_relative "benchmark-cadence" -options = {} -OptionParser.new do |parser| - parser.on("--case ID") { |value| options[:case] = value } - parser.on("--output PATH") { |value| options[:output] = value } -end.parse! -raise "Use --output for the proposal inventory" unless options[:output] -items = options[:case] ? [BenchmarkCases.load_case(options[:case])] : BenchmarkCases.documents -items = items.select { |item| item.dig("execution", "sync") != "fixed" && item.dig("execution", "sync") && item.dig("execution", "blockers").to_a.empty? } -records = items.map do |item| - begin - BenchmarkCases.sync_source(item).merge("state" => "proposal", "requires_case_qualification" => true) - rescue BenchmarkCases::Error => error - {"case_id" => item.fetch("id"), "state" => "blocked", "error" => error.message} +module SourceSync + def self.propose(items, output:, sync: BenchmarkCases.method(:sync_source)) + inventory = {"schema_version" => 1, "records" => [], "publication" => "requires-review"} + BenchmarkCases.write_json(output, inventory) + items.each do |item| + record = begin + blockers = item.dig("execution", "blockers").to_a + raise BenchmarkCases::Error, blockers.join("; ") unless blockers.empty? + proposal = sync.call(item) + proposal.merge("state" => proposal.fetch("updated") ? "proposal" : "unchanged", + "requires_case_qualification" => proposal.fetch("updated")) + rescue BenchmarkCases::Error => error + {"case_id" => item.fetch("id"), "state" => "blocked", "error" => error.message} + end + inventory.fetch("records") << record + BenchmarkCases.write_json(output, inventory) + puts "#{record.fetch('case_id')}: #{record.fetch('state')}" + $stdout.flush + end + inventory end end -BenchmarkCases.validate -BenchmarkCases.write_json(options.fetch(:output), {"schema_version" => 1, "records" => records, "publication" => "requires-review"}) -puts "Prepared #{records.length} source proposals; #{records.count { |record| record['state'] == 'blocked' }} blocked" -exit(records.any? { |record| record["state"] == "blocked" } ? 1 : 0) + +if $PROGRAM_NAME == __FILE__ + options = {} + OptionParser.new do |parser| + parser.on("--case ID") { |value| options[:case] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--matrix") { options[:matrix] = true } + end.parse! + items = BenchmarkCadence.source_cases(case_id: options[:case]) + if options[:matrix] + puts JSON.generate({"case_id" => items.map { |item| item.fetch("id") }}) + exit + end + abort "Use --output for the proposal inventory" unless options[:output] + inventory = SourceSync.propose(items, output: options.fetch(:output)) + BenchmarkCases.validate + exit(inventory.fetch("records").any? { |record| record["state"] == "blocked" } ? 1 : 0) +end diff --git a/scripts/verify-upstream-recipe.rb b/scripts/verify-upstream-recipe.rb index 06c276de8..80e20ee9b 100755 --- a/scripts/verify-upstream-recipe.rb +++ b/scripts/verify-upstream-recipe.rb @@ -3,11 +3,21 @@ require_relative "benchmark-plan" require "digest" +require "open3" root = BenchmarkPlan::ROOT source = ARGV.first || File.join(root, "upstream") contract = JSON.parse(File.read(File.join(root, ARGV[1] || "recipe-contract.json"))) -contract.fetch("upstream_files").each do |path, digest| +files = contract.fetch("upstream_files") +if contract.key?("upstream_files_by_revision") + revision, status = Open3.capture2("git", "-C", source, "rev-parse", "HEAD") + raise "Cannot determine the source revision for recipe verification" unless status.success? + overrides = contract.fetch("upstream_files_by_revision") + raise "Recipe exceptions require exact source revisions" unless overrides.keys.all? { |sha| sha.match?(/\A[0-9a-f]{40}\z/) } + files = overrides.fetch(revision.strip, files) + raise "Recipe exception changes the reviewed file set" unless files.keys.sort == contract.fetch("upstream_files").keys.sort +end +files.each do |path, digest| actual = File.join(source, path) raise "Upstream recipe file is missing: #{path}" unless File.file?(actual) raise "Upstream recipe changed: #{path}; review the workload before updating its recipe digest" unless Digest::SHA256.file(actual).hexdigest == digest diff --git a/suites/scheduled.json b/suites/scheduled.json new file mode 100644 index 000000000..fe19e1c67 --- /dev/null +++ b/suites/scheduled.json @@ -0,0 +1,103 @@ +{ + "schema_version": 1, + "cases": [ + { + "case_id": "hugo" + }, + { + "case_id": "hugo-go" + }, + { + "case_id": "immich" + }, + { + "case_id": "mastodon", + "variant": "server" + }, + { + "case_id": "mastodon", + "variant": "streaming" + }, + { + "case_id": "posthog", + "variant": "layers" + }, + { + "case_id": "storybook" + }, + { + "case_id": "opentelemetry-java" + }, + { + "case_id": "spring-ai" + }, + { + "case_id": "grpc" + }, + { + "case_id": "zed", + "variant": "combined" + }, + { + "case_id": "deno", + "variant": "cargo-product" + }, + { + "case_id": "duckgres" + }, + { + "case_id": "chroma" + }, + { + "case_id": "linkerd2" + }, + { + "case_id": "qdrant" + }, + { + "case_id": "n8n", + "variant": "turbo" + }, + { + "case_id": "n8n", + "variant": "docker" + }, + { + "case_id": "n8n", + "variant": "runners" + }, + { + "case_id": "n8n", + "variant": "distroless" + }, + { + "case_id": "gogs-moon" + }, + { + "case_id": "opencut-moon" + }, + { + "case_id": "stackstorm-pants" + }, + { + "case_id": "executorch-buck2" + }, + { + "case_id": "msgpack-sbt" + }, + { + "case_id": "helix-nix" + }, + { + "case_id": "zed-nix" + }, + { + "case_id": "obs-studio", + "variant": "ccache" + }, + { + "case_id": "obs-studio", + "variant": "xcode" + } + ] +} diff --git a/test/benchmark_cadence_test.rb b/test/benchmark_cadence_test.rb new file mode 100644 index 000000000..e4bf648ec --- /dev/null +++ b/test/benchmark_cadence_test.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../scripts/benchmark-cadence" +require_relative "../scripts/sync-sources" + +class BenchmarkCadenceTest < Minitest::Test + class Runner < NightlyCanaries::Runner + attr_reader :dispatches + + def initialize + @dispatches = [] + end + + def api(path, body: nil) + if body + @dispatches << {"path" => path, "body" => body} + return {"workflow_run_id" => @dispatches.length} + end + return {"state" => "active"} if path.include?("/actions/workflows/") + stable = path.end_with?("/latest") + release = {"tag_name" => stable ? "v1.40.0" : "vcli-canary-0123456789ab", "draft" => false, + "prerelease" => !stable, "published_at" => "2026-10-05T00:00:00Z", + "assets" => %w[SHA256SUMS boringcache-linux-amd64 boringcache-linux-arm64 boringcache-macos-universal boringcache-windows-amd64.exe].map { |name| {"name" => name} }} + stable ? release : [release] + end + end + + def test_weekly_and_nightly_dispatch_the_same_workloads_with_exact_release_tags + targets = BenchmarkCadence.fresh_targets + runners = %w[stable canary].map do |channel| + runner = Runner.new + Dir.mktmpdir do |directory| + record = runner.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: targets, + channel: channel, output: File.join(directory, "receipt.json")) + assert_equal "requested", record.fetch("state") + assert_equal targets.length, record.fetch("runs").length + end + runner + end + stable, canary = runners.map(&:dispatches) + assert_equal stable.map { |run| run.fetch("path") }, canary.map { |run| run.fetch("path") } + stable.zip(canary).each do |weekly, nightly| + assert_equal "v1.40.0", weekly.dig("body", "inputs", "cli_version") + assert_equal "vcli-canary-0123456789ab", nightly.dig("body", "inputs", "cli_version") + assert_equal weekly.dig("body", "inputs").except("cli_version"), nightly.dig("body", "inputs").except("cli_version") + end + end + + def test_scheduled_suite_includes_maintained_cases_and_the_new_families + scheduled = BenchmarkCadence.cases.map { |entry| entry.fetch("case_id") } + maintained = JSON.parse(File.read(File.join(BenchmarkCases::ROOT, "suites/release.json"))).fetch("cases") + assert_empty maintained - scheduled + assert_empty %w[gogs-moon opencut-moon stackstorm-pants executorch-buck2 msgpack-sbt helix-nix zed-nix] - scheduled + refute_includes scheduled, "pants-jvm" + refute scheduled.any? { |id| id.start_with?("docker-") } + end + + def test_shared_workflows_keep_case_and_variant_selectors + targets = BenchmarkCadence.fresh_targets + native = targets.select { |item| item.fetch("fresh_workflow") == "reapi-fresh-benchmark.yml" } + assert_equal %w[executorch-buck2 gogs-moon msgpack-sbt opencut-moon stackstorm-pants], native.map { |item| item.dig("fresh_inputs", "case_id") }.sort + n8n = targets.select { |item| item.fetch("case_id") == "n8n" } + assert_equal %w[distroless docker runners turbo], n8n.map { |item| item.dig("fresh_inputs", "variant") }.sort + obs = targets.select { |item| item.fetch("case_id") == "obs-studio" } + assert_equal 4, obs.length + assert_equal 4, NightlyCanaries::Runner.new.targets(obs).length + end + + def test_source_checks_use_the_same_suite_once_per_case + ids = BenchmarkCadence.cases.map { |entry| entry.fetch("case_id") }.uniq + assert_equal ids, BenchmarkCadence.source_cases.map { |item| item.fetch("id") } + assert_equal ["helix-nix"], BenchmarkCadence.source_cases(case_id: "helix-nix").map { |item| item.fetch("id") } + assert_raises(BenchmarkCadence::Error) { BenchmarkCadence.source_cases(case_id: "pants-jvm") } + end + + def test_source_matrix_selects_each_case_once_and_rejects_unknown_cases + script = File.join(BenchmarkCases::ROOT, "scripts/sync-sources.rb") + output, status = Open3.capture2e(RbConfig.ruby, script, "--matrix") + assert status.success?, output + ids = JSON.parse(output).fetch("case_id") + assert_equal BenchmarkCadence.source_cases.map { |item| item.fetch("id") }, ids + assert_equal ids.uniq, ids + output, status = Open3.capture2e(RbConfig.ruby, script, "--matrix", "--case", "helix-nix") + assert status.success?, output + assert_equal({"case_id" => ["helix-nix"]}, JSON.parse(output)) + output, status = Open3.capture2e(RbConfig.ruby, script, "--matrix", "--case", "unknown") + refute status.success? + assert_includes output, "not in the scheduled suite" + end + + def test_source_inventory_retains_unchanged_changed_and_blocked_cases + items = %w[hugo-go gogs-moon helix-nix].map { |id| BenchmarkCases.load_case(id) } + sync = lambda do |item| + id = item.fetch("id") + raise BenchmarkCases::Error, "Source recipe changed" if id == "helix-nix" + {"case_id" => id, "updated" => id == "hugo-go"} + end + Dir.mktmpdir do |directory| + path = File.join(directory, "sources.json") + inventory = nil + capture_io { inventory = SourceSync.propose(items, output: path, sync: sync) } + assert_equal inventory, JSON.parse(File.read(path)) + assert_equal %w[proposal unchanged blocked], inventory.fetch("records").map { |record| record.fetch("state") } + assert_equal false, inventory.fetch("records")[1].fetch("requires_case_qualification") + assert_equal "Source recipe changed", inventory.fetch("records")[2].fetch("error") + end + end + + def test_native_rolling_paths_verify_outputs_for_every_scheduled_variant + %w[hugo chroma duckgres linkerd2 immich mastodon posthog n8n].each do |id| + targets = BenchmarkCadence.rolling_targets(case_id: id) + selections = BenchmarkCadence.cases.select { |entry| entry.fetch("case_id") == id } + assert_equal selections.length, targets.length, id + assert targets.all? { |target| target.fetch("workflow") == "native-rolling-benchmark.yml" } + assert_equal selections.map { |entry| entry["variant"] }, targets.map { |target| target.dig("inputs", "variant") } + end + %w[gogs-moon opencut-moon stackstorm-pants executorch-buck2 msgpack-sbt helix-nix zed-nix].each do |id| + targets = BenchmarkCadence.rolling_targets(case_id: id) + assert_equal 1, targets.length + assert_equal id, targets.first.dig("inputs", "case_id") + assert_equal "rolling", targets.first.fetch("lane") + end + end + + def test_schedules_require_explicit_cutover_and_preserve_historical_monitoring + %w[weekly-fresh canary source-sync].each do |name| + workflow = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, ".github/workflows/#{name}.yml"))) + job = workflow.fetch("jobs").values.first + assert_equal "github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true'", job.fetch("if") + end + workflow = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, ".github/workflows/nightly-canaries.yml"))) + step = workflow.dig("jobs", "results", "steps").find { |item| item["name"] == "Check repository canaries" } + assert_includes step.fetch("run"), 'scripts/nightly-canaries.rb --collect' + assert_includes step.fetch("run"), 'scripts/benchmark-cadence.rb --collect' + end + + def test_incompatible_cli_stops_the_entire_dispatch_and_retains_a_failed_receipt + runner = Runner.new + preflight = ->(version, runs) { BenchmarkCadence.verify_cli(version, runs, probe: ->(*) { "--port PORT" }) } + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + error = assert_raises(BenchmarkCadence::Error) do + runner.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: BenchmarkCadence.fresh_targets, + channel: "stable", output: path, preflight: preflight) + end + assert_includes error.message, "--reapi-port" + assert_empty runner.dispatches + receipt = JSON.parse(File.read(path)) + assert_equal "dispatch-failed", receipt.fetch("state") + assert_equal 31, receipt.fetch("runs").length + assert receipt.fetch("runs").all? { |run| run.fetch("state") == "planned" } + end + end + + def test_cli_capability_probe_is_required_for_reapi_but_not_other_targets + runs = [{"workflow" => "reapi-fresh-benchmark.yml"}] + assert_nil BenchmarkCadence.verify_cli("v1.34.0", runs, probe: ->(*) { "--reapi-port PORT" }) + assert_nil BenchmarkCadence.verify_cli("v1.33.0", [{"workflow" => "native-fresh-benchmark.yml"}], + probe: ->(*) { flunk "No REAPI capability is needed" }) + end +end diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 6d264525d..4802f38ea 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -338,6 +338,41 @@ def test_selected_recipe_contract_checks_each_declared_plan end end + def test_parent_recipe_exception_is_limited_to_its_exact_revision_and_file_set + with_case("hugo-go") do |directory| + source = File.join(directory, "upstream") + FileUtils.mkdir_p(source) + git = ->(*args) { Open3.capture2e("git", "-C", source, "-c", "user.name=Benchmark test", "-c", "user.email=benchmark@localhost", "-c", "commit.gpgsign=false", *args) } + _, status = git.call("init") + assert status.success? + File.write(File.join(source, "recipe.txt"), "parent recipe\n") + git.call("add", ".") + _, status = git.call("commit", "-m", "Parent recipe") + assert status.success? + revision, = git.call("rev-parse", "HEAD") + parent = {"recipe.txt" => Digest::SHA256.hexdigest("parent recipe\n")} + contract = {"upstream_files" => {"recipe.txt" => Digest::SHA256.hexdigest("new recipe\n")}, + "upstream_files_by_revision" => {revision.strip => parent}, "commands" => {}} + path = File.join(directory, "recipe-contract.json") + File.write(path, JSON.generate(contract)) + _, error, status = run_script(directory, "verify-upstream-recipe") + assert status.success?, error + git.call("commit", "--allow-empty", "-m", "Another revision") + _, error, status = run_script(directory, "verify-upstream-recipe") + refute status.success? + assert_includes error, "Upstream recipe changed" + File.write(File.join(source, "recipe.txt"), "new recipe\n") + _, error, status = run_script(directory, "verify-upstream-recipe") + assert status.success?, error + current, = git.call("rev-parse", "HEAD") + contract["upstream_files_by_revision"] = {current.strip => {}} + File.write(path, JSON.generate(contract)) + _, error, status = run_script(directory, "verify-upstream-recipe") + refute status.success? + assert_includes error, "changes the reviewed file set" + end + end + def test_deno_profile_selects_exactly_one_declared_profile with_case("deno") do |directory| path = File.join(directory, ".boringcache.toml") diff --git a/test/benchmark_cases_test.rb b/test/benchmark_cases_test.rb index 6550f63aa..aa3cadcc2 100644 --- a/test/benchmark_cases_test.rb +++ b/test/benchmark_cases_test.rb @@ -168,6 +168,66 @@ def test_source_advancement_preserves_other_reviewed_source_pairs end end + def test_snapshot_advancement_verifies_before_changing_pins_or_nix_commands + Dir.mktmpdir do |root| + FileUtils.mkdir_p(File.join(root, "cases")) + FileUtils.cp_r(File.join(BenchmarkCases::ROOT, "cases/helix-nix"), File.join(root, "cases")) + item = BenchmarkCases.load_case("helix-nix", root) + base = item.dig("source", "revision") + head = "a" * 40 + calls = [] + command = lambda do |*args, **| + calls << args + if args.first == "gh" + args.last.include?("/compare/") ? JSON.generate({"status" => "ahead", "merge_base_commit" => {"sha" => base}}) : JSON.generate({"sha" => head}) + else + assert_equal base, BenchmarkCases.load_case("helix-nix", root).dig("source", "revision") + "Verified" + end + end + prepare = lambda do |candidate, **| + assert_equal head, candidate.dig("source", "revision") + assert_equal [head], candidate.dig("source", "pins").map { |pin| pin.fetch("revision") } + root + end + BenchmarkCases.stub(:command, command) do + BenchmarkCases.stub(:prepare, prepare) do + result = BenchmarkCases.sync_source(item, root: root) + assert_equal base, result.fetch("base_sha") + assert_equal head, result.fetch("head_sha") + end + end + assert_equal head, BenchmarkCases.load_case("helix-nix", root).dig("source", "revision") + assert_equal base, item.dig("source", "revision"), "Inspection must not mutate the original declaration" + payload = File.join(root, "cases/helix-nix/payload") + assert_includes File.read(File.join(payload, ".boringcache.toml")), "/#{head}#default" + contract = JSON.parse(File.read(File.join(payload, "recipe-contract.json"))) + original = JSON.parse(File.read(File.join(BenchmarkCases::ROOT, "cases/helix-nix/payload/recipe-contract.json"))) + assert_equal original.fetch("upstream_files"), contract.fetch("upstream_files") + assert calls.any? { |args| args.include?(File.join(root, "scripts/verify-upstream-recipe.rb")) } + end + end + + def test_snapshot_recipe_failure_leaves_definition_and_payload_unchanged + Dir.mktmpdir do |root| + FileUtils.mkdir_p(File.join(root, "cases")) + FileUtils.cp_r(File.join(BenchmarkCases::ROOT, "cases/helix-nix"), File.join(root, "cases")) + item = BenchmarkCases.load_case("helix-nix", root) + files = Dir.glob(File.join(root, "cases/helix-nix/**/{*,.*}")).select { |path| File.file?(path) } + original = files.to_h { |path| [path, File.binread(path)] } + command = lambda do |*args, **| + raise BenchmarkCases::Error, "Recipe changed" unless args.first == "gh" + args.last.include?("/compare/") ? JSON.generate({"status" => "ahead", "merge_base_commit" => {"sha" => item.dig("source", "revision")}}) : JSON.generate({"sha" => "a" * 40}) + end + BenchmarkCases.stub(:command, command) do + BenchmarkCases.stub(:prepare, ->(*, **) { root }) do + assert_raises(BenchmarkCases::Error) { BenchmarkCases.sync_source(item, root: root) } + end + end + assert_equal original, files.to_h { |path| [path, File.binread(path)] } + end + end + def test_source_sync_fetches_the_declared_branch_from_a_single_branch_shallow_checkout with_root do |root| git = ->(*args, directory:) { BenchmarkCases.command("git", "-c", "user.name=Benchmark test", "-c", "user.email=benchmark@localhost", "-c", "commit.gpgsign=false", *args, chdir: directory) } diff --git a/test/benchmark_phase_test.rb b/test/benchmark_phase_test.rb index aff806cce..5448f1a35 100644 --- a/test/benchmark_phase_test.rb +++ b/test/benchmark_phase_test.rb @@ -60,4 +60,14 @@ def test_warm_consumption_never_exports_and_explicit_publication_applies_to_both refute_includes File.read(path), "docker/build-push-action@", "#{path}: Docker provider lifecycle must be shared" end end + + def test_explicit_rolling_cohort_links_frozen_source_generations_without_changing_fresh_identity + env = {"BENCHMARK_ID" => "posthog", "CACHE_LANE" => "rolling", "BENCHMARK_SERIES_ID" => "parent-01", + "BENCHMARK_SAMPLE" => "1", "BENCHMARK_ROLLING_SCOPE" => "qualification-01", "GITHUB_RUN_ID" => "123", "GITHUB_RUN_ATTEMPT" => "1"} + assert_equal "posthog-rolling-qualification-01", BenchmarkPhase.scope(env) + assert_equal BenchmarkPhase.scope(env), BenchmarkPhase.scope(env.merge("BENCHMARK_SERIES_ID" => "child-01", "GITHUB_RUN_ID" => "456")) + fresh = env.merge("CACHE_LANE" => "fresh") + assert_equal BenchmarkPhase.scope(fresh.reject { |key, _| key == "BENCHMARK_ROLLING_SCOPE" }), BenchmarkPhase.scope(fresh) + assert_raises(RuntimeError) { BenchmarkPhase.scope(env.merge("BENCHMARK_ROLLING_SCOPE" => "../other")) } + end end diff --git a/test/cadence_dispatch_test.rb b/test/cadence_dispatch_test.rb new file mode 100644 index 000000000..22c7b1f39 --- /dev/null +++ b/test/cadence_dispatch_test.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../scripts/cadence-dispatch" + +class CadenceDispatchTest < Minitest::Test + def plan + {"state" => "planned", "cli_version" => "vcli-canary-0123456789ab", "channel" => "canary", "ref" => "main", + "runs" => %w[immich posthog].map { |id| {"repository" => "boringcache/benchmarks", "workflow" => "native-fresh-benchmark.yml", "inputs" => {"case_id" => id}, "state" => "planned"} }} + end + + def dispatch(index, response: {"workflow_run_id" => 123}) + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + runner = NightlyCanaries::Runner.new + request = lambda do |endpoint, body:| + assert_equal "requesting", JSON.parse(File.read(path)).dig("runs", 0, "state") + assert_equal "repos/boringcache/benchmarks/actions/workflows/native-fresh-benchmark.yml/dispatches", endpoint + assert_equal plan.fetch("cli_version"), body.dig("inputs", "cli_version") + assert_equal plan.dig("runs", index, "inputs", "case_id"), body.dig("inputs", "case_id") + response + end + runner.stub(:api, request) do + begin + result = CadenceDispatch.dispatch(plan, index: index, output: path, runner: runner) + assert_equal result, JSON.parse(File.read(path)) + result + rescue CadenceDispatch::Error + JSON.parse(File.read(path)) + end + end + end + end + + def test_independent_dispatch_receipts_combine_in_declared_order + first, second = dispatch(0), dispatch(1, response: {"workflow_run_id" => 456}) + combined = CadenceDispatch.combine(plan, [second, first]) + assert_equal "requested", combined.fetch("state") + assert_equal [123, 456], combined.fetch("runs").map { |run| run.fetch("id") } + assert_equal "planned", plan.fetch("state") + assert_equal [0, 1], CadenceDispatch.matrix(plan).fetch("include").map { |row| row.fetch("index") } + end + + def test_ambiguous_request_and_missing_target_are_retained + unknown = dispatch(0, response: {}) + assert_equal "dispatch-failed", unknown.fetch("state") + assert_equal "request-unknown", unknown.dig("runs", 0, "state") + combined = CadenceDispatch.combine(plan, [unknown]) + assert_equal "dispatch-failed", combined.fetch("state") + assert_equal %w[request-unknown planned], combined.fetch("runs").map { |run| run.fetch("state") } + end + + def test_failed_request_does_not_remove_successful_sibling + failed = dispatch(0, response: {}) + success = dispatch(1) + combined = CadenceDispatch.combine(plan, [failed, success]) + assert_equal "dispatch-failed", combined.fetch("state") + assert_equal %w[request-unknown requested], combined.fetch("runs").map { |run| run.fetch("state") } + end + + def test_duplicate_mismatched_and_incomplete_receipts_cannot_claim_success + receipt = dispatch(0) + assert_raises(CadenceDispatch::Error) { CadenceDispatch.combine(plan, [receipt, receipt]) } + wrong_version = receipt.merge("cli_version" => "v1.33.0") + assert_raises(CadenceDispatch::Error) { CadenceDispatch.combine(plan, [wrong_version]) } + wrong_index = receipt.merge("target_index" => 1) + assert_raises(CadenceDispatch::Error) { CadenceDispatch.combine(plan, [wrong_index]) } + assert_equal "dispatch-failed", CadenceDispatch.combine(plan, [receipt]).fetch("state") + assert_equal "planned", CadenceDispatch.combine(plan, [], dry_run: true).fetch("state") + assert_raises(CadenceDispatch::Error) { CadenceDispatch.matrix(plan.merge("state" => "dispatch-failed")) } + end +end diff --git a/test/native_case_test.rb b/test/native_case_test.rb index 45c165932..c50818b9e 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -13,13 +13,14 @@ def with_payload(id) end end - def test_fresh_and_rolling_share_the_recipe_but_preserve_publication_behavior + def test_fresh_and_rolling_share_the_recipe_and_loaded_output with_payload("hugo") do |item, directory| fresh = NativeCase.write_action(item, directory: directory, lane: "fresh") rolling = NativeCase.write_action(item, directory: directory, lane: "rolling") assert_equal fresh.dig("runs", "steps", 0, "uses"), rolling.dig("runs", "steps", 0, "uses") assert_equal "true", fresh.dig("runs", "steps", 0, "with", "load_image") - assert_equal "true", rolling.dig("runs", "steps", 0, "with", "push_image") + assert_equal "true", rolling.dig("runs", "steps", 0, "with", "load_image") + refute rolling.dig("runs", "steps", 0, "with").key?("push_image") assert_equal "${{ inputs.cli_version }}", fresh.dig("runs", "steps", 0, "with", "cli_version") end end @@ -129,6 +130,8 @@ def test_native_planning_rejects_missing_output_verification_before_dispatch %w[chroma duckgres hugo linkerd2].each do |id| item = BenchmarkCases.load_case(id) assert_equal "native-fresh-benchmark.yml", BenchmarkCases.plan(item).fetch("workflow") + assert_equal "native-rolling-benchmark.yml", BenchmarkCases.plan(item, lane: "rolling").fetch("workflow") + item.fetch("execution").fetch("native").fetch("rolling_inputs").delete("load_image") error = assert_raises(NativeCase::Error) { BenchmarkCases.plan(item, lane: "rolling") } assert_includes error.message, "requires load_image=true" end diff --git a/test/nix_benchmark_test.rb b/test/nix_benchmark_test.rb index c00b8146c..68fc8325c 100644 --- a/test/nix_benchmark_test.rb +++ b/test/nix_benchmark_test.rb @@ -18,6 +18,22 @@ def test_ambiguous_provider_configuration_fails assert_raises(NixBenchmark::Error) { NixBenchmark.provider_substituter(config, "boringcache", "") } end + def test_changed_source_can_build_after_a_provider_miss_but_warm_cannot + config = {"substituters" => {"value" => ["https://cache.nixos.org", "https://benchmark.cachix.org"]}} + BenchmarkPlan.stub(:command, %w[nix build pinned-source]) do + NixBenchmark.stub(:capture, JSON.generate(config)) do + rolling = NixBenchmark.build_command(phase: "commit", provider: "cachix", cache_name: "benchmark") + warm = NixBenchmark.build_command(phase: "warm", provider: "cachix", cache_name: "benchmark") + assert_includes rolling, "https://benchmark.cachix.org" + refute_includes rolling, "https://cache.nixos.org" + refute_includes rolling, "--max-jobs" + assert_equal rolling + ["--max-jobs", "0", "--builders", ""], warm + assert_equal %w[nix build pinned-source --option substitute false], NixBenchmark.build_command(phase: "cold", provider: "cachix") + assert_raises(NixBenchmark::Error) { NixBenchmark.build_command(phase: "invalid", provider: "cachix") } + end + end + end + def test_preexisting_package_output_cannot_be_reported_as_a_warm_restore Dir.mktmpdir do |directory| Dir.chdir(directory) do @@ -47,4 +63,26 @@ def test_comparison_rejects_changed_dependency_and_output_hashes assert_raises(NixBenchmark::Error) { NixBenchmark.compare(seed: seed, current: current) } end end + + def test_dependency_archive_is_verified_before_import + Dir.mktmpdir do |directory| + File.write(File.join(directory, "store.nar.zst"), "changed") + NixBenchmark.write(File.join(directory, "archive.json"), {"sha256" => Digest::SHA256.hexdigest("expected")}) + Open3.stub(:pipeline, ->(*) { flunk "Unverified archive was imported" }) do + error = assert_raises(NixBenchmark::Error) { NixBenchmark.import_dependencies(directory: directory) } + assert_includes error.message, "checksum differs" + end + end + end + + def test_dependency_export_cannot_seed_the_measured_package + Dir.mktmpdir do |directory| + Dir.chdir(directory) do + NixBenchmark.write("nix-evidence/baseline.json", {"outputs" => ["/nix/store/package"], "dependencies" => {"/nix/store/package" => "hash"}}) + Open3.stub(:pipeline, ->(*) { flunk "Measured package was exported as a dependency" }) do + assert_raises(NixBenchmark::Error) { NixBenchmark.export_dependencies } + end + end + end + end end diff --git a/test/reapi_registry_test.rb b/test/reapi_registry_test.rb index cb8ebc592..09efc5bf7 100644 --- a/test/reapi_registry_test.rb +++ b/test/reapi_registry_test.rb @@ -31,6 +31,16 @@ def test_warm_comparator_requires_authentication_for_writes assert_includes args, "--allow_unauthenticated_reads" end + def test_changed_source_registries_allow_reads_and_publication + args = ReapiRegistry.command(provider: "boringcache", phase: "commit", workspace: "boringcache/benchmarks", tag: "case-series") + refute_includes args, "--read-only" + assert_includes args, "--fail-on-cache-error" + args = ReapiRegistry.command(provider: "bazel-remote", phase: "commit", workspace: "boringcache/benchmarks", tag: "case-series") + refute_includes args, "--htpasswd_file" + assert_includes args, "reapi-store" + assert_raises(ReapiRegistry::Error) { ReapiRegistry.command(provider: "boringcache", phase: "unknown", workspace: "boringcache/benchmarks", tag: "case-series") } + end + def test_failed_shutdown_fails_after_a_successful_build Dir.mktmpdir do |directory| server = TCPServer.new("127.0.0.1", 0) diff --git a/test/reapi_report_test.rb b/test/reapi_report_test.rb index 4294ef96f..275eedeba 100644 --- a/test/reapi_report_test.rb +++ b/test/reapi_report_test.rb @@ -13,6 +13,8 @@ def test_phase_name_does_not_establish_a_cache_hit_or_miss assert_nil reported_cache_hit("warm", "0 remote cache hits") assert_equal true, reported_cache_hit("cold", "2 remote cache hits") assert_equal true, reported_cache_hit("warm", "81 remote cache hits") + assert_nil reported_cache_hit("commit", "0 remote cache hits") + assert_equal true, reported_cache_hit("commit", "2 remote cache hits") end private @@ -20,8 +22,8 @@ def test_phase_name_does_not_establish_a_cache_hit_or_miss def reported_cache_hit(phase, log) previous = ENV.to_h ENV.update("PHASE" => phase, "PROVIDER" => "bazel-remote", "BENCHMARK_ID" => "msgpack-sbt", - "CACHE_LANE" => "fresh", "BENCHMARK_SAMPLE" => "1", - "GITHUB_RUN_ID" => "1", "GITHUB_RUN_ATTEMPT" => "1") + "CACHE_LANE" => phase == "commit" ? "rolling" : "fresh", "BENCHMARK_SAMPLE" => "1", + "GITHUB_RUN_ID" => "1", "GITHUB_RUN_ATTEMPT" => "1", "GITHUB_REF_NAME" => "main") ENV.delete("BENCHMARK_SERIES_ID") Dir.mktmpdir do |directory| Dir.chdir(directory) do diff --git a/test/source_promotion_test.rb b/test/source_promotion_test.rb new file mode 100644 index 000000000..bc18da236 --- /dev/null +++ b/test/source_promotion_test.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../scripts/source-promotion" + +class SourcePromotionTest < Minitest::Test + VERSION = "vcli-canary-0123456789ab" + + def proposal(item) + prefix = item.dig("execution", "source_prefix") + base = if prefix + BenchmarkPlan.settings(File.join(BenchmarkCases::ROOT, "cases", item.fetch("id"), "payload/benchmark-source.env")).fetch("#{prefix}_HEAD_SHA") + else + item.dig("source", "revision") || item.dig("source", "pins").find { |pin| pin["kind"] == "gitlink" }.fetch("revision") + end + record = {"case_id" => item.fetch("id"), "state" => "proposal", "base_sha" => base, "head_sha" => "a" * 40, + "source_distance" => "1", "requires_verified_build" => item.dig("execution", "sync") == "verified-pair"} + if prefix + text = File.read(File.join(BenchmarkCases::ROOT, "cases", item.fetch("id"), "payload/benchmark-source.env")) + record["candidate_source_env"] = text.sub(/^#{prefix}_BASE_SHA=.*$/, "#{prefix}_BASE_SHA=#{base}").sub(/^#{prefix}_HEAD_SHA=.*$/, "#{prefix}_HEAD_SHA=#{'a' * 40}") + end + record + end + + class Publisher < SourcePromotion::Publisher + attr_reader :saved, :requests + attr_accessor :fail_request, :outcome + + def initialize + super + @saved, @requests, @files = [], [], {} + @outcome = "in_progress" + end + + def optional_file(path, ref: "main") + @files[path] + end + + def save(record, expected:, changes: {}) + path = state_path(record.fetch("case_id")) + raise "State was overwritten" unless expected.fetch(path) == @files[path] + @saved << {"record" => Marshal.load(Marshal.dump(record)), "changes" => changes} + @files[path] = JSON.pretty_generate(record) + "\n" + end + + def api(path, body: nil) + if body + raise "Dispatch happened before a durable intent" unless @saved.last.dig("record", "state") == "dispatching" + @requests << body + return {} if @fail_request == @requests.length + {"workflow_run_id" => @requests.length} + else + {"status" => @outcome == "in_progress" ? "in_progress" : "completed", "conclusion" => @outcome} + end + end + end + + def test_snapshot_promotion_changes_only_source_pins_and_matching_nix_uri + item = BenchmarkCases.load_case("helix-nix") + candidate, changes = SourcePromotion.changes(item, proposal(item)) + assert_equal "a" * 40, candidate.dig("source", "revision") + assert_equal item.except("source"), candidate.except("source") + assert_equal %w[cases/helix-nix/case.json cases/helix-nix/payload/.boringcache.toml cases/helix-nix/payload/recipe-contract.json], changes.keys.sort + old = JSON.parse(File.read(File.join(BenchmarkCases::ROOT, "cases/helix-nix/payload/recipe-contract.json"))) + actual = JSON.parse(changes.fetch("cases/helix-nix/payload/recipe-contract.json")) + assert_equal old.fetch("upstream_files"), actual.fetch("upstream_files") + end + + def test_source_environment_cannot_change_toolchain_repository_or_commands + item = BenchmarkCases.load_case("deno") + proposed = proposal(item) + SourcePromotion.changes(item, proposed) + %w[DENO_RUST_VERSION DENO_SOURCE_REPOSITORY].each do |key| + changed = proposed.merge("candidate_source_env" => proposed.fetch("candidate_source_env").sub(/^#{key}=.*$/, "#{key}=changed")) + assert_raises(SourcePromotion::Error) { SourcePromotion.changes(item, changed) } + end + assert_raises(SourcePromotion::Error) { SourcePromotion.changes(item, proposed.merge("base_sha" => "b" * 40)) } + end + + def test_native_variants_and_zed_auto_are_the_only_selected_rolling_targets + item = BenchmarkCases.load_case("n8n") + targets = SourcePromotion.targets(item, proposal(item), version: VERSION) + assert_equal %w[distroless docker runners turbo], targets.map { |target| target.dig("inputs", "variant") }.sort + assert targets.all? { |target| target.dig("inputs", "cli_version") == VERSION } + zed = BenchmarkCases.load_case("zed") + targets = SourcePromotion.targets(zed, proposal(zed), version: VERSION) + assert_equal ["zed-zed-cargo-rolling-auto.yml"], targets.map { |target| target.fetch("workflow") } + end + + def test_every_scheduled_case_can_plan_a_changed_source_run + BenchmarkCadence.source_cases.each do |item| + targets = SourcePromotion.targets(item, proposal(item), version: VERSION) + refute_empty targets, item.fetch("id") + assert targets.all? { |target| target.fetch("lane") == "rolling" }, item.fetch("id") + end + end + + def test_proposal_cannot_disable_zed_build_verification + item = BenchmarkCases.load_case("zed") + assert_raises(SourcePromotion::Error) { SourcePromotion.changes(item, proposal(item).merge("requires_verified_build" => false)) } + end + + def test_request_failure_retains_other_variants_and_blocks_blind_retry + item = BenchmarkCases.load_case("n8n") + publisher = Publisher.new + publisher.fail_request = 2 + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + assert_raises(SourcePromotion::Error) { publisher.publish(item, proposal(item), version: VERSION, output: path) } + assert_equal 4, publisher.requests.length + record = JSON.parse(File.read(path)) + assert_equal "dispatch-failed", record.fetch("state") + assert_equal %w[requested request-unknown requested requested], record.fetch("runs").map { |run| run.fetch("state") } + assert publisher.saved.first.fetch("changes").key?("cases/n8n/case.json") + assert_raises(SourcePromotion::Error) { publisher.publish(item, proposal(item), version: VERSION, output: path) } + assert_equal 4, publisher.requests.length + end + end + + def test_zed_promotes_only_after_all_requested_builds_succeed + item = BenchmarkCases.load_case("zed") + publisher = Publisher.new + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + publisher.publish(item, proposal(item), version: VERSION, output: path) + assert_empty publisher.saved.first.fetch("changes") + assert_equal "requested", publisher.reconcile(item, output: path).fetch("state") + assert_equal 2, publisher.saved.length + publisher.outcome = "success" + assert_equal "promoted", publisher.reconcile(item, output: path).fetch("state") + assert publisher.saved.last.fetch("changes").key?("cases/zed/case.json") + assert publisher.saved.last.fetch("changes").key?("cases/zed/payload/benchmark-source.env") + end + end + + def test_failed_zed_build_preserves_the_declared_source + item = BenchmarkCases.load_case("zed") + publisher = Publisher.new + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + publisher.publish(item, proposal(item), version: VERSION, output: path) + publisher.outcome = "failure" + assert_equal "failed", publisher.reconcile(item, output: path).fetch("state") + assert publisher.saved.all? { |entry| entry.fetch("changes").empty? } + assert_raises(SourcePromotion::Error) { publisher.publish(item, proposal(item), version: VERSION, output: path) } + end + end + + def test_publication_rejects_a_concurrent_edit_instead_of_overwriting_it + publisher = SourcePromotion::Publisher.new + requests = [] + api = lambda do |path, body: nil| + requests << path + path.include?("git/ref") ? {"object" => {"sha" => "b" * 40}} : {"type" => "file", "content" => Base64.strict_encode64("edited")} + end + publisher.stub(:api, api) do + assert_raises(SourcePromotion::Error) { publisher.commit({"cases/example/case.json" => "new"}, expected: {"cases/example/case.json" => "original"}, message: "Advance source") } + end + refute_includes requests, "graphql" + end + + def test_unrelated_branch_advancement_retries_with_a_new_expected_head + publisher = SourcePromotion::Publisher.new + heads = ["b" * 40, "c" * 40] + commits = [] + api = lambda do |path, body: nil| + if path == "graphql" + commits << body.fetch("variables").fetch("input") + commits.length == 1 ? {"errors" => [{"type" => "STALE_DATA", "message" => "Branch advanced"}]} : {"data" => {"createCommitOnBranch" => {"commit" => {"oid" => "d" * 40}}}} + elsif path.include?("git/ref") + {"object" => {"sha" => heads.shift}} + else + {"type" => "file", "content" => Base64.strict_encode64("original")} + end + end + publisher.stub(:api, api) do + assert_equal "d" * 40, publisher.commit({"cases/example/case.json" => "new"}, expected: {"cases/example/case.json" => "original"}, message: "Advance source") + end + assert_equal ["b" * 40, "c" * 40], commits.map { |input| input.fetch("expectedHeadOid") } + assert_equal "new", Base64.decode64(commits.last.dig("fileChanges", "additions", 0, "contents")) + end +end