From 7be3cf64ab78b117dd476ce3aca6afc5acc01b75 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jul 2026 21:56:18 +0000 Subject: [PATCH 1/2] Benchmark PRs on machines chosen by machine: labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR benchmark always ran on c6a.4xlarge, but some systems cannot work there — e.g. Hyrise (#883), an in-memory DBMS whose encoded hits table alone exceeds the machine's 32 GB. The manual workflow can target any machine, but its runs are invisible on the PR until the results sink picks them up, and every re-run needs a manual dispatch. A maintainer can now add one or more `machine:` labels to a PR (e.g. `machine:c6a.metal`) to override the default machine, one run per label. Adding such a label relaunches the benchmark immediately; other labels don't trigger anything. Labels can only be set by users with triage access, and every launch stays gated by the `benchmark-approval` environment, so the cost control is unchanged. Co-Authored-By: Claude Fable 5 --- .github/workflows/README.md | 2 +- .github/workflows/benchmark-pr.yml | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8c5714c45..c43aa6178 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -15,7 +15,7 @@ apart from the runs of main and are excluded by `collect-results.sh`. |----------|---------|------------------| | `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | | `benchmark-manual.yml` | manual | any systems, machines, repository and branch | -| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval | +| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label); adding such a label relaunches the benchmark | | `collect-results.yml` | every 30 minutes | nothing - it collects the runs of the last day from the sink database (`collect-new-results.py`): commits result files and posts pastila.nl log links to the corresponding PR, or maintains one automated results PR per system for the runs of main | ## Setup diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index fc1d4ad7f..7ff7a46c4 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -6,6 +6,10 @@ name: "Benchmark pull request" # disposable benchmark machines, which clone the PR's repository and branch. on: pull_request_target: + # `labeled` lets a maintainer relaunch the benchmark on a different + # machine by adding a "machine:" label; the detect job filters + # out all other labels so they don't relaunch anything. + types: [opened, synchronize, reopened, labeled] permissions: contents: read @@ -18,11 +22,17 @@ concurrency: jobs: detect: name: Detect changed systems + # Unlike the other trigger types, `labeled` can also fire on a closed PR, + # hence the state guard. + if: >- + github.event.pull_request.state == 'open' && + (github.event.action != 'labeled' || startsWith(github.event.label.name, 'machine:')) runs-on: ubuntu-latest permissions: pull-requests: read outputs: systems: ${{ steps.changed.outputs.systems }} + machines: ${{ steps.changed.outputs.machines }} steps: - id: changed env: @@ -51,6 +61,18 @@ jobs: echo "Systems to benchmark: $systems" echo "systems=$systems" >> "$GITHUB_OUTPUT" + # "machine:" labels override the default machine, e.g. + # machine:c6a.metal for systems that don't fit in the 32 GB of + # c6a.4xlarge. Multiple labels launch one run per machine. Labels + # can only be set by users with triage access, and the launch is + # still gated by the benchmark-approval environment. + machines=$(jq -r '[.pull_request.labels[].name + | select(startswith("machine:")) | ltrimstr("machine:")] + | join(" ")' "$GITHUB_EVENT_PATH") + machines="${machines:-c6a.4xlarge}" + echo "Machines: $machines" + echo "machines=$machines" >> "$GITHUB_OUTPUT" + launch: needs: detect if: needs.detect.outputs.systems != '' @@ -67,6 +89,7 @@ jobs: - uses: ./.github/actions/launch-benchmark with: systems: ${{ needs.detect.outputs.systems }} + machines: ${{ needs.detect.outputs.machines }} # The head repository and branch to benchmark are derived from the # PR number by run-benchmark.sh. pr: ${{ github.event.pull_request.number }} From c714fb816521184b0aaadc18a7cd78dfb4c789ce Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jul 2026 22:00:35 +0000 Subject: [PATCH 2/2] Expand machine:all / all-amd / all-arm label aliases `machine:all`, `machine:all-amd` and `machine:all-arm` labels expand to the machine sets of the daily runs (the lists mirror benchmark-daily.yml), deduplicated against explicitly labeled machines. Co-Authored-By: Claude Fable 5 --- .github/workflows/README.md | 2 +- .github/workflows/benchmark-pr.yml | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c43aa6178..af93b34ce 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -15,7 +15,7 @@ apart from the runs of main and are excluded by `collect-results.sh`. |----------|---------|------------------| | `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | | `benchmark-manual.yml` | manual | any systems, machines, repository and branch | -| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label); adding such a label relaunches the benchmark | +| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label; `machine:all`, `machine:all-amd` and `machine:all-arm` expand to the daily-run machine sets); adding such a label relaunches the benchmark | | `collect-results.yml` | every 30 minutes | nothing - it collects the runs of the last day from the sink database (`collect-new-results.py`): commits result files and posts pastila.nl log links to the corresponding PR, or maintains one automated results PR per system for the runs of main | ## Setup diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 7ff7a46c4..99a1ac20c 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -63,12 +63,28 @@ jobs: # "machine:" labels override the default machine, e.g. # machine:c6a.metal for systems that don't fit in the 32 GB of - # c6a.4xlarge. Multiple labels launch one run per machine. Labels - # can only be set by users with triage access, and the launch is - # still gated by the benchmark-approval environment. - machines=$(jq -r '[.pull_request.labels[].name + # c6a.4xlarge. Multiple labels launch one run per machine, and + # machine:all, machine:all-amd and machine:all-arm expand to the + # machine sets of the daily runs (keep in sync with + # benchmark-daily.yml). Labels can only be set by users with + # triage access, and the launch is still gated by the + # benchmark-approval environment. + amd="t3a.small c6a.large c6a.xlarge c6a.2xlarge c6a.4xlarge c6a.metal c7a.metal-48xl" + arm="c8g.4xlarge c8g.metal-48xl" + labels=$(jq -r '[.pull_request.labels[].name | select(startswith("machine:")) | ltrimstr("machine:")] | join(" ")' "$GITHUB_EVENT_PATH") + machines="" + for label in $labels; do + case "$label" in + all) machines="$machines $amd $arm";; + all-amd) machines="$machines $amd";; + all-arm) machines="$machines $arm";; + *) machines="$machines $label";; + esac + done + machines=$(printf '%s\n' $machines | awk '!seen[$0]++' | tr '\n' ' ') + machines="${machines% }" machines="${machines:-c6a.4xlarge}" echo "Machines: $machines" echo "machines=$machines" >> "$GITHUB_OUTPUT"