Share benchmark suite across weekly and nightly runs

.github/actions/nix-benchmark/action.yml

@@ -7,44 +7,43 @@ inputs:
     required: true
   phase:
     required: true
+  lane:
+    default: fresh
   cli-version:
     default: ''
   cachix-cache:
     default: ''
   cachix-token:
     default: ''
+  dependency-seed:
+    default: ''
 runs:
   using: composite
   steps:
-    - name: Verify the source recipe
-      shell: bash
-      run: ruby scripts/verify-upstream-recipe.rb .
-    - name: Install Nix
-      uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24
-      with:
-        install_url: https://releases.nixos.org/nix/nix-2.35.2/install
-        enable_kvm: false
-        extra_nix_config: |
-          accept-flake-config = false
-          experimental-features = nix-command flakes
+    - uses: ./.github/actions/nix-environment
     - name: Prepare the dependency store
       shell: bash
+      env:
+        DEPENDENCY_SEED: ${{ inputs.dependency-seed }}
       run: |
-        test "$(nix --version)" = 'nix (Nix) 2.35.2'
-        ruby scripts/nix-benchmark.rb prepare
+        if [[ -n "$DEPENDENCY_SEED" ]]; then
+          ruby scripts/nix-benchmark.rb import-dependencies "$DEPENDENCY_SEED"
+        else
+          ruby scripts/nix-benchmark.rb prepare
+        fi
     - name: Scope the cache and start setup timing
       id: setup
       shell: bash
       env:
         BENCHMARK_ID: ${{ inputs.case-id }}
-        CACHE_LANE: fresh
+        CACHE_LANE: ${{ inputs.lane }}
       run: |
         scope="$(ruby scripts/benchmark-phase.rb scope)"
         ruby scripts/scope-case-cache.rb "$scope"
         echo "started_at=$(date +%s.%N)" >> "$GITHUB_OUTPUT"
     - name: Set up BoringCache
       id: publish
-      if: inputs.provider == 'boringcache' && inputs.phase == 'cold'
+      if: inputs.provider == 'boringcache' && inputs.phase != 'warm'
       uses: ./.github/actions/boringcache
       with:
         mode: nix
@@ -104,17 +103,20 @@ runs:
         BENCHMARK_ID: ${{ inputs.case-id }}
         PROVIDER: ${{ inputs.provider }}
         PHASE: ${{ inputs.phase }}
+        CACHE_LANE: ${{ inputs.lane }}
         SETUP_SECONDS: ${{ steps.timing.outputs.seconds }}
         EVIDENCE_PATH: ${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path }}
       run: |
         source_repository="$(ruby -rjson -e 'puts JSON.parse(File.read("benchmark-context.json")).dig("source", "repository")')"
         source_sha="$(ruby -rjson -e 'puts JSON.parse(File.read("benchmark-context.json")).dig("source", "revision")')"
         build_seconds="$(ruby -rjson -e 'puts JSON.parse(File.read("nix-evidence/timing.json")).fetch("build_seconds")')"
+        args=()
+        [[ "$PHASE" != warm ]] || args+=(--cache-hit true)
+        [[ "$PHASE" != cold ]] || args+=(--cache-hit false)
         ruby scripts/benchmark-report.rb phase --verified-output \
-          --benchmark "$BENCHMARK_ID" --strategy "$PROVIDER" --lane fresh --phase "$PHASE" --mode nix \
+          --benchmark "$BENCHMARK_ID" --strategy "$PROVIDER" --lane "$CACHE_LANE" --phase "$PHASE" --mode nix \
           --restore-or-setup-seconds "$SETUP_SECONDS" --build-seconds "$build_seconds" \
-          --cache-hit "$([[ "$PHASE" == warm ]] && echo true || echo false)" \
-          --source-repository "$source_repository" --source-sha "$source_sha" --evidence "$EVIDENCE_PATH"
+          --source-repository "$source_repository" --source-sha "$source_sha" --evidence "$EVIDENCE_PATH" "${args[@]}"
     - name: Retain Nix checks and logs
       if: always()
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
@@ -125,6 +127,6 @@ runs:
     - name: Retain phase measurements
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
       with:
-        name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-fresh-${{ inputs.phase }}
+        name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-${{ inputs.lane }}-${{ inputs.phase }}
         path: benchmark-results/
         if-no-files-found: error

.github/actions/nix-environment/action.yml

@@ -0,0 +1,19 @@
+name: Prepare Nix benchmark environment
+description: Verifies the declared recipe and installs the pinned Nix version.
+runs:
+  using: composite
+  steps:
+    - name: Verify the source recipe
+      shell: bash
+      run: ruby scripts/verify-upstream-recipe.rb .
+    - name: Install Nix
+      uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24
+      with:
+        install_url: https://releases.nixos.org/nix/nix-2.35.2/install
+        enable_kvm: false
+        extra_nix_config: |
+          accept-flake-config = false
+          experimental-features = nix-command flakes
+    - name: Verify the Nix version
+      shell: bash
+      run: test "$(nix --version)" = 'nix (Nix) 2.35.2'

.github/actions/reapi-benchmark/action.yml

@@ -9,6 +9,8 @@ inputs:
     required: true
   cli-version:
     required: true
+  lane:
+    default: fresh
 runs:
   using: composite
   steps:
@@ -35,6 +37,22 @@ runs:
       with:
         name: reapi-store-${{ inputs.case-id }}
         path: reapi-store
+    - name: Restore the rolling comparator store
+      if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling'
+      id: rolling-cache
+      uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae
+      with:
+        path: reapi-store
+        key: reapi-${{ inputs.case-id }}-${{ env.BENCHMARK_ROLLING_SCOPE || env.BENCHMARK_SERIES_ID || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          reapi-${{ inputs.case-id }}-${{ env.BENCHMARK_ROLLING_SCOPE || env.BENCHMARK_SERIES_ID || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}-
+    - name: Record the restored comparator seed
+      if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling'
+      shell: bash
+      env:
+        BENCHMARK_ID: ${{ inputs.case-id }}
+        RESTORED_CACHE_KEY: ${{ steps.rolling-cache.outputs.cache-matched-key }}
+      run: ruby scripts/reapi-setup.rb rolling-seed
     - name: Download cold output checks
       if: inputs.phase == 'warm'
       uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53
@@ -48,7 +66,7 @@ runs:
         PHASE: ${{ inputs.phase }}
         PROVIDER: ${{ inputs.provider }}
         BENCHMARK_ID: ${{ inputs.case-id }}
-        CACHE_LANE: fresh
+        CACHE_LANE: ${{ inputs.lane }}
         BORINGCACHE_WORKSPACE: boringcache/benchmarks
       run: boringcache ci run --oidc-provider github-actions -- ruby scripts/reapi-client.rb build
     - name: Run bazel-remote
@@ -58,18 +76,28 @@ runs:
         PHASE: ${{ inputs.phase }}
         PROVIDER: ${{ inputs.provider }}
         BENCHMARK_ID: ${{ inputs.case-id }}
-        CACHE_LANE: fresh
+        CACHE_LANE: ${{ inputs.lane }}
       run: ruby scripts/reapi-client.rb build
     - name: Verify outputs and record measurements
       shell: bash
       env:
         PHASE: ${{ inputs.phase }}
         PROVIDER: ${{ inputs.provider }}
         BENCHMARK_ID: ${{ inputs.case-id }}
-        CACHE_LANE: fresh
+        CACHE_LANE: ${{ inputs.lane }}
       run: |
         ruby scripts/reapi-client.rb verify
         ruby scripts/reapi-setup.rb report
+    - name: Record the verified comparator source
+      if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling'
+      shell: bash
+      run: ruby scripts/reapi-setup.rb publish-seed
+    - name: Publish the verified rolling comparator store
+      if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling'
+      uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae
+      with:
+        path: reapi-store
+        key: ${{ steps.rolling-cache.outputs.cache-primary-key }}
     - name: Retain the comparator store
       if: inputs.provider == 'bazel-remote' && inputs.phase == 'cold'
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
@@ -88,6 +116,6 @@ runs:
     - name: Retain phase measurements
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
       with:
-        name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-fresh-${{ inputs.phase }}
+        name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-${{ inputs.lane }}-${{ inputs.phase }}
         path: benchmark-results/
         if-no-files-found: error

.github/workflows/canary.yml

@@ -1,38 +1,28 @@
 name: Dispatch CLI Canary Cases
 on:
+  schedule:
+    - cron: '17 1 * * *'
   workflow_dispatch:
     inputs:
       cli_version:
         description: Optional exact published CLI canary; empty selects the latest
         type: string
         default: ''
+      dry_run:
+        description: Validate the dispatch without starting builds.
+        type: boolean
+        default: false
 permissions:
   contents: read
+  actions: write
 concurrency:
   group: benchmark-canary-dispatch
   cancel-in-progress: false
 jobs:
   dispatch:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 15
-    permissions:
-      contents: read
-      actions: write
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
-        with:
-          persist-credentials: false
-      - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b
-      - name: Dispatch registered cases
-        env:
-          GH_TOKEN: ${{ github.token }}
-          CLI_VERSION: ${{ inputs.cli_version }}
-        run: ruby scripts/nightly-canaries.rb --repository "$GITHUB_REPOSITORY" --version "$CLI_VERSION" --output "$RUNNER_TEMP/nightly-canaries.json" --summary "$GITHUB_STEP_SUMMARY"
-      - name: Retain dispatch receipt
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
-        with:
-          name: nightly-canaries
-          path: ${{ runner.temp }}/nightly-canaries.json
-          if-no-files-found: error
-          retention-days: 90
+    if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true'
+    uses: ./.github/workflows/fresh-cadence.yml
+    with:
+      channel: canary
+      cli_version: ${{ inputs.cli_version || '' }}
+      dry_run: ${{ inputs.dry_run || false }}

.github/workflows/fresh-cadence.yml

@@ -0,0 +1,136 @@
+name: Dispatch fresh benchmark runs
+on:
+  workflow_call:
+    inputs:
+      channel:
+        type: string
+        required: true
+      cli_version:
+        type: string
+        default: ''
+      dry_run:
+        type: boolean
+        default: false
+permissions:
+  contents: read
+  actions: read
+jobs:
+  plan:
+    name: Validate suite and CLI
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+        with:
+          persist-credentials: false
+      - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b
+      - run: bundle install --jobs 4 --retry 2
+      - name: Validate every target before requesting builds
+        env:
+          GH_TOKEN: ${{ github.token }}
+          CHANNEL: ${{ inputs.channel }}
+          CLI_VERSION: ${{ inputs.cli_version }}
+        run: |
+          ruby scripts/benchmark-cadence.rb --dry-run \
+            --channel "$CHANNEL" --version "$CLI_VERSION" --ref "$GITHUB_REF_NAME" \
+            --output "$RUNNER_TEMP/benchmark-dispatch.json" \
+            --summary "$GITHUB_STEP_SUMMARY"
+      - id: matrix
+        run: |
+          echo "matrix=$(ruby scripts/cadence-dispatch.rb --plan "$RUNNER_TEMP/benchmark-dispatch.json" --matrix)" >> "$GITHUB_OUTPUT"
+      - name: Retain validated plan
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: benchmark-plan
+          path: ${{ runner.temp }}/benchmark-dispatch.json
+          if-no-files-found: error
+          retention-days: 90
+      - name: Retain failed preflight receipt
+        if: failure()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: benchmark-dispatch
+          path: ${{ runner.temp }}/benchmark-dispatch.json
+          if-no-files-found: error
+          retention-days: 90
+  dispatch:
+    name: '${{ matrix.label }} / ${{ inputs.channel }} fresh'
+    needs: plan
+    if: '!inputs.dry_run'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      actions: write
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+        with:
+          persist-credentials: false
+      - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b
+      - run: bundle install --jobs 4 --retry 2
+      - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53
+        with:
+          name: benchmark-plan
+          path: ${{ runner.temp }}/plan
+      - name: Request this benchmark
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TARGET_INDEX: ${{ matrix.index }}
+        run: |
+          ruby scripts/cadence-dispatch.rb \
+            --plan "$RUNNER_TEMP/plan/benchmark-dispatch.json" --index "$TARGET_INDEX" \
+            --output "$RUNNER_TEMP/target-$TARGET_INDEX.json"
+      - name: Retain this request
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: benchmark-target-${{ matrix.index }}
+          path: ${{ runner.temp }}/target-${{ matrix.index }}.json
+          if-no-files-found: error
+          retention-days: 90
+  receipt:
+    name: Collect requested runs
+    needs: [plan, dispatch]
+    if: always() && needs.plan.result == 'success'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+        with:
+          persist-credentials: false
+      - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b
+      - run: bundle install --jobs 4 --retry 2
+      - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53
+        with:
+          name: benchmark-plan
+          path: ${{ runner.temp }}/plan
+      - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53
+        if: '!inputs.dry_run'
+        with:
+          pattern: benchmark-target-*
+          path: ${{ runner.temp }}/receipts
+      - name: Combine requests without losing failed or missing targets
+        if: always()
+        env:
+          DRY_RUN: ${{ inputs.dry_run }}
+        run: |
+          args=()
+          [[ "$DRY_RUN" != true ]] || args+=(--dry-run)
+          ruby scripts/cadence-dispatch.rb \
+            --plan "$RUNNER_TEMP/plan/benchmark-dispatch.json" \
+            --receipts "$RUNNER_TEMP/receipts" \
+            --output "$RUNNER_TEMP/benchmark-dispatch.json" \
+            --summary "$GITHUB_STEP_SUMMARY" "${args[@]}"
+      - name: Retain dispatch receipt
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: benchmark-dispatch
+          path: ${{ runner.temp }}/benchmark-dispatch.json
+          if-no-files-found: error
+          retention-days: 90

.github/workflows/guardrails.yml

@@ -16,6 +16,8 @@ jobs:
       run: bundle exec ruby bin/bench check
     - name: Check publication registry
       run: bundle exec ruby scripts/check-registry-alignment.rb
+    - name: Check scheduled cases
+      run: bundle exec ruby scripts/benchmark-cadence.rb --check
     - name: Check execution contracts
       run: bundle exec ruby scripts/check-workflow-guardrails.rb
     - name: Check report contracts

.github/workflows/native-fresh-benchmark.yml

@@ -1,6 +1,6 @@
 ---
 name: Native comparison fresh
-run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample
+run-name: "${{ inputs.case_id }} ${{ inputs.variant }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample
   }}"
 permissions:
   contents: read