diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8c5714c45..af93b34ce 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -15,7 +15,7 @@ apart from the runs of main and are excluded by `collect-results.sh`. |----------|---------|------------------| | `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | | `benchmark-manual.yml` | manual | any systems, machines, repository and branch | -| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval | +| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval. A `machine:` label overrides the default c6a.4xlarge (one run per label; `machine:all`, `machine:all-amd` and `machine:all-arm` expand to the daily-run machine sets); adding such a label relaunches the benchmark | | `collect-results.yml` | every 30 minutes | nothing - it collects the runs of the last day from the sink database (`collect-new-results.py`): commits result files and posts pastila.nl log links to the corresponding PR, or maintains one automated results PR per system for the runs of main | ## Setup diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index fc1d4ad7f..99a1ac20c 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -6,6 +6,10 @@ name: "Benchmark pull request" # disposable benchmark machines, which clone the PR's repository and branch. on: pull_request_target: + # `labeled` lets a maintainer relaunch the benchmark on a different + # machine by adding a "machine:" label; the detect job filters + # out all other labels so they don't relaunch anything. + types: [opened, synchronize, reopened, labeled] permissions: contents: read @@ -18,11 +22,17 @@ concurrency: jobs: detect: name: Detect changed systems + # Unlike the other trigger types, `labeled` can also fire on a closed PR, + # hence the state guard. + if: >- + github.event.pull_request.state == 'open' && + (github.event.action != 'labeled' || startsWith(github.event.label.name, 'machine:')) runs-on: ubuntu-latest permissions: pull-requests: read outputs: systems: ${{ steps.changed.outputs.systems }} + machines: ${{ steps.changed.outputs.machines }} steps: - id: changed env: @@ -51,6 +61,34 @@ jobs: echo "Systems to benchmark: $systems" echo "systems=$systems" >> "$GITHUB_OUTPUT" + # "machine:" labels override the default machine, e.g. + # machine:c6a.metal for systems that don't fit in the 32 GB of + # c6a.4xlarge. Multiple labels launch one run per machine, and + # machine:all, machine:all-amd and machine:all-arm expand to the + # machine sets of the daily runs (keep in sync with + # benchmark-daily.yml). Labels can only be set by users with + # triage access, and the launch is still gated by the + # benchmark-approval environment. + amd="t3a.small c6a.large c6a.xlarge c6a.2xlarge c6a.4xlarge c6a.metal c7a.metal-48xl" + arm="c8g.4xlarge c8g.metal-48xl" + labels=$(jq -r '[.pull_request.labels[].name + | select(startswith("machine:")) | ltrimstr("machine:")] + | join(" ")' "$GITHUB_EVENT_PATH") + machines="" + for label in $labels; do + case "$label" in + all) machines="$machines $amd $arm";; + all-amd) machines="$machines $amd";; + all-arm) machines="$machines $arm";; + *) machines="$machines $label";; + esac + done + machines=$(printf '%s\n' $machines | awk '!seen[$0]++' | tr '\n' ' ') + machines="${machines% }" + machines="${machines:-c6a.4xlarge}" + echo "Machines: $machines" + echo "machines=$machines" >> "$GITHUB_OUTPUT" + launch: needs: detect if: needs.detect.outputs.systems != '' @@ -67,6 +105,7 @@ jobs: - uses: ./.github/actions/launch-benchmark with: systems: ${{ needs.detect.outputs.systems }} + machines: ${{ needs.detect.outputs.machines }} # The head repository and branch to benchmark are derived from the # PR number by run-benchmark.sh. pr: ${{ github.event.pull_request.number }}