From 6c9850f813f1264f7fb5ee3afba8748113488e39 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jul 2026 02:38:31 +0000 Subject: [PATCH 1/3] Add GitHub Actions to launch benchmark machines on AWS Three workflows around the existing run-benchmark.sh launcher, using a federated IAM role via OIDC. The GitHub-hosted runner only launches the self-terminating EC2 machine; the machine runs the benchmark unattended and sends its log to the results sink as before. - benchmark-daily.yml: the six ClickHouse variants on c6a.4xlarge, daily. - benchmark-manual.yml: any system/machine/repo/branch on demand. - benchmark-pr.yml: systems whose directories a PR changes (ignoring results and *.md), from the PR's repo/branch, gated by required reviewers on the benchmark-approval environment. pull_request_target keeps the launcher scripts on the trusted base branch; PR code only runs on the disposable machine. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/actions/launch-benchmark/action.yml | 62 +++++++++++++++++ .github/workflows/README.md | 50 +++++++++++++ .github/workflows/benchmark-daily.yml | 35 ++++++++++ .github/workflows/benchmark-manual.yml | 42 +++++++++++ .github/workflows/benchmark-pr.yml | 77 +++++++++++++++++++++ 5 files changed, 266 insertions(+) create mode 100644 .github/actions/launch-benchmark/action.yml create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/benchmark-daily.yml create mode 100644 .github/workflows/benchmark-manual.yml create mode 100644 .github/workflows/benchmark-pr.yml diff --git a/.github/actions/launch-benchmark/action.yml b/.github/actions/launch-benchmark/action.yml new file mode 100644 index 0000000000..eab9cc437b --- /dev/null +++ b/.github/actions/launch-benchmark/action.yml @@ -0,0 +1,62 @@ +name: "Launch benchmark" +description: > + Assume the federated AWS IAM role and launch an EC2 machine that runs one + benchmark unattended. The runner only launches: run-benchmark.sh renders + cloud-init.sh.in and calls ec2 run-instances; the machine clones the + repository, runs /benchmark.sh, sends its log to the results sink + and terminates itself. + +inputs: + system: + description: "System to benchmark: a directory name, e.g. clickhouse-parquet" + required: true + machine: + description: "EC2 instance type" + default: "c6a.4xlarge" + repo: + description: "GitHub repository (owner/name) the machine will clone" + default: "ClickHouse/ClickBench" + branch: + description: "Branch of that repository to clone" + default: "main" + timeout: + description: "Time limit for benchmark.sh on the machine, seconds" + default: "36000" + role-to-assume: + description: "ARN of the federated IAM role that can launch EC2 machines" + required: true + aws-region: + description: "AWS region to launch the machine in" + required: true + +runs: + using: "composite" + steps: + - id: check + shell: bash + env: + role: ${{ inputs.role-to-assume }} + run: | + if [ -z "$role" ]; then + echo "The BENCHMARK_AWS_ROLE_ARN repository variable is not set. Skipping." + echo "configured=false" >> "$GITHUB_OUTPUT" + else + echo "configured=true" >> "$GITHUB_OUTPUT" + fi + + - if: steps.check.outputs.configured == 'true' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: ${{ inputs.aws-region }} + role-session-name: clickbench-${{ github.run_id }} + + - if: steps.check.outputs.configured == 'true' + shell: bash + env: + system: ${{ inputs.system }} + machine: ${{ inputs.machine }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + timeout: ${{ inputs.timeout }} + run: ./run-benchmark.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..fa46d4de5e --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,50 @@ +# Benchmark automation + +These workflows launch benchmark machines on AWS. The GitHub-hosted runner is +used only to launch: it assumes a federated IAM role and runs +`run-benchmark.sh`, which starts a self-terminating EC2 machine. The machine +clones the repository, runs `/benchmark.sh` under cloud-init (see +`cloud-init.sh.in`), sends its log to the results sink at play.clickhouse.com, +and shuts down. Collecting the results back into `/results/` is a +separate process, implemented separately. + +| Workflow | Trigger | What it launches | +|----------|---------|------------------| +| `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants on c6a.4xlarge from main | +| `benchmark-manual.yml` | manual | any system, machine, repository and branch | +| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval | + +## Setup + +1. An IAM role for GitHub's OIDC provider, restricted to this repository: + + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { "Federated": "arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com" }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" }, + "StringLike": { "token.actions.githubusercontent.com:sub": "repo:ClickHouse/ClickBench:*" } + } + }] + } + ``` + + The permissions policy needs `ec2:RunInstances`, `ec2:CreateTags`, + `ec2:DescribeImages`, and `ec2:DescribeInstanceTypes`. + +2. Repository variables `BENCHMARK_AWS_ROLE_ARN` (the ARN of that role) and + `BENCHMARK_AWS_REGION`. While the variables are not set, the workflows + skip the launch instead of failing. + +3. An environment named `benchmark-approval` with required reviewers. It + gates the PR workflow: nothing is launched for a pull request until a + reviewer approves the pending deployment. + +4. Enough on-demand vCPU quota in the region: the daily run launches six + c6a.4xlarge machines at once, 96 vCPUs, on top of whatever is still + running. `run-benchmark.sh` waits and retries while the quota is + exhausted, but only within the job's 55-minute limit. diff --git a/.github/workflows/benchmark-daily.yml b/.github/workflows/benchmark-daily.yml new file mode 100644 index 0000000000..a457ee1f4f --- /dev/null +++ b/.github/workflows/benchmark-daily.yml @@ -0,0 +1,35 @@ +name: "Daily benchmarks: ClickHouse" + +on: + workflow_dispatch: # This allows manual trigger from the UI + schedule: + - cron: '0 2 * * *' + +permissions: + id-token: write # To assume the federated IAM role. + contents: read + +jobs: + launch: + name: ${{ matrix.system }} + runs-on: ubuntu-latest + # The AWS session lasts an hour; run-benchmark.sh retries while the vCPU + # quota is exhausted, so cap the job before the credentials expire. + timeout-minutes: 55 + strategy: + fail-fast: false + matrix: + system: + - clickhouse + - clickhouse-web + - clickhouse-parquet + - clickhouse-parquet-partitioned + - clickhouse-datalake + - clickhouse-datalake-partitioned + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/launch-benchmark + with: + system: ${{ matrix.system }} + role-to-assume: ${{ vars.BENCHMARK_AWS_ROLE_ARN }} + aws-region: ${{ vars.BENCHMARK_AWS_REGION }} diff --git a/.github/workflows/benchmark-manual.yml b/.github/workflows/benchmark-manual.yml new file mode 100644 index 0000000000..00e7a162db --- /dev/null +++ b/.github/workflows/benchmark-manual.yml @@ -0,0 +1,42 @@ +name: "Run a benchmark" + +run-name: "Benchmark ${{ inputs.system }} on ${{ inputs.machine }} from ${{ inputs.repo }}@${{ inputs.branch }}" + +on: + workflow_dispatch: + inputs: + system: + description: "System to benchmark (a directory name)" + required: true + machine: + description: "EC2 instance type" + default: "c6a.4xlarge" + repo: + description: "Repository to clone on the machine" + default: "ClickHouse/ClickBench" + branch: + description: "Branch to clone" + default: "main" + timeout: + description: "Time limit for benchmark.sh, seconds" + default: "36000" + +permissions: + id-token: write # To assume the federated IAM role. + contents: read + +jobs: + launch: + runs-on: ubuntu-latest + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/launch-benchmark + with: + system: ${{ inputs.system }} + machine: ${{ inputs.machine }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + timeout: ${{ inputs.timeout }} + role-to-assume: ${{ vars.BENCHMARK_AWS_ROLE_ARN }} + aws-region: ${{ vars.BENCHMARK_AWS_REGION }} diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml new file mode 100644 index 0000000000..70c0df9223 --- /dev/null +++ b/.github/workflows/benchmark-pr.yml @@ -0,0 +1,77 @@ +name: "Benchmark pull request" + +# pull_request_target: the workflow and the launcher scripts are always taken +# from the trusted base branch, and the OIDC token is available for PRs from +# forks. Code from the PR never runs on the runner - it only runs on the +# disposable benchmark machine, which clones the PR's repository and branch. +on: + pull_request_target: + +permissions: + contents: read + +# A new push supersedes queued runs, including ones still waiting for approval. +concurrency: + group: benchmark-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + detect: + name: Detect changed systems + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + systems: ${{ steps.changed.outputs.systems }} + steps: + - id: changed + env: + GH_TOKEN: ${{ github.token }} + BASE_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + # A system should be benchmarked if the PR touches files in its + # directory - except results and documentation - and the directory + # contains benchmark.sh at the PR head, so that added systems are + # included and deleted ones are not. + systems='[]' + if [ -n "$HEAD_REPO" ]; then + dirs=$(gh api "repos/$BASE_REPO/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename' \ + | grep -v -E '(^|/)results/|\.md$' \ + | awk -F/ 'NF > 1 { print $1 }' | sort -u || true) + for dir in $dirs; do + if gh api --silent "repos/$HEAD_REPO/contents/$dir/benchmark.sh?ref=$HEAD_SHA" 2>/dev/null; then + systems=$(jq -c --arg dir "$dir" '. + [$dir]' <<< "$systems") + fi + done + fi + echo "Systems to benchmark: $systems" + echo "systems=$systems" >> "$GITHUB_OUTPUT" + + launch: + name: ${{ matrix.system }} + needs: detect + if: needs.detect.outputs.systems != '[]' + runs-on: ubuntu-latest + # Launching machines for PR code costs money and the code is untrusted: + # this environment must be configured with required reviewers. + environment: benchmark-approval + permissions: + id-token: write # To assume the federated IAM role. + contents: read + timeout-minutes: 55 + strategy: + fail-fast: false + matrix: + system: ${{ fromJSON(needs.detect.outputs.systems) }} + steps: + - uses: actions/checkout@v4 # The base branch: trusted launcher scripts. + - uses: ./.github/actions/launch-benchmark + with: + system: ${{ matrix.system }} + repo: ${{ github.event.pull_request.head.repo.full_name }} + branch: ${{ github.event.pull_request.head.ref }} + role-to-assume: ${{ vars.BENCHMARK_AWS_ROLE_ARN }} + aws-region: ${{ vars.BENCHMARK_AWS_REGION }} From 08bab9295e04f989404a11e26563b3710546a83e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jul 2026 02:58:43 +0000 Subject: [PATCH 2/3] Hardcode the IAM role and default to us-east-1 The role ARN and region were repository variables passed through every workflow. The role is a single fixed thing, so set it directly in the composite action; the region defaults to us-east-1 (cheaper). The variables and the skip-when-unconfigured check are gone. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/actions/launch-benchmark/action.yml | 27 +++++---------------- .github/workflows/README.md | 9 +++---- .github/workflows/benchmark-daily.yml | 2 -- .github/workflows/benchmark-manual.yml | 2 -- .github/workflows/benchmark-pr.yml | 2 -- 5 files changed, 10 insertions(+), 32 deletions(-) diff --git a/.github/actions/launch-benchmark/action.yml b/.github/actions/launch-benchmark/action.yml index eab9cc437b..9ba5234762 100644 --- a/.github/actions/launch-benchmark/action.yml +++ b/.github/actions/launch-benchmark/action.yml @@ -22,37 +22,22 @@ inputs: timeout: description: "Time limit for benchmark.sh on the machine, seconds" default: "36000" - role-to-assume: - description: "ARN of the federated IAM role that can launch EC2 machines" - required: true aws-region: description: "AWS region to launch the machine in" - required: true + default: "us-east-1" runs: using: "composite" steps: - - id: check - shell: bash - env: - role: ${{ inputs.role-to-assume }} - run: | - if [ -z "$role" ]; then - echo "The BENCHMARK_AWS_ROLE_ARN repository variable is not set. Skipping." - echo "configured=false" >> "$GITHUB_OUTPUT" - else - echo "configured=true" >> "$GITHUB_OUTPUT" - fi - - - if: steps.check.outputs.configured == 'true' - uses: aws-actions/configure-aws-credentials@v4 + # The role's trust policy only admits workflow runs of this repository, + # so assuming it fails everywhere else, including forks. + - uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ inputs.role-to-assume }} + role-to-assume: arn:aws:iam::264113785604:role/ClickBenchLauncher aws-region: ${{ inputs.aws-region }} role-session-name: clickbench-${{ github.run_id }} - - if: steps.check.outputs.configured == 'true' - shell: bash + - shell: bash env: system: ${{ inputs.system }} machine: ${{ inputs.machine }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index fa46d4de5e..9ff661c855 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -36,15 +36,14 @@ separate process, implemented separately. The permissions policy needs `ec2:RunInstances`, `ec2:CreateTags`, `ec2:DescribeImages`, and `ec2:DescribeInstanceTypes`. -2. Repository variables `BENCHMARK_AWS_ROLE_ARN` (the ARN of that role) and - `BENCHMARK_AWS_REGION`. While the variables are not set, the workflows - skip the launch instead of failing. + The role's ARN and the region (us-east-1) are set in + `.github/actions/launch-benchmark/action.yml`. -3. An environment named `benchmark-approval` with required reviewers. It +2. An environment named `benchmark-approval` with required reviewers. It gates the PR workflow: nothing is launched for a pull request until a reviewer approves the pending deployment. -4. Enough on-demand vCPU quota in the region: the daily run launches six +3. Enough on-demand vCPU quota in the region: the daily run launches six c6a.4xlarge machines at once, 96 vCPUs, on top of whatever is still running. `run-benchmark.sh` waits and retries while the quota is exhausted, but only within the job's 55-minute limit. diff --git a/.github/workflows/benchmark-daily.yml b/.github/workflows/benchmark-daily.yml index a457ee1f4f..d27cd79ddb 100644 --- a/.github/workflows/benchmark-daily.yml +++ b/.github/workflows/benchmark-daily.yml @@ -31,5 +31,3 @@ jobs: - uses: ./.github/actions/launch-benchmark with: system: ${{ matrix.system }} - role-to-assume: ${{ vars.BENCHMARK_AWS_ROLE_ARN }} - aws-region: ${{ vars.BENCHMARK_AWS_REGION }} diff --git a/.github/workflows/benchmark-manual.yml b/.github/workflows/benchmark-manual.yml index 00e7a162db..746a6d58f1 100644 --- a/.github/workflows/benchmark-manual.yml +++ b/.github/workflows/benchmark-manual.yml @@ -38,5 +38,3 @@ jobs: repo: ${{ inputs.repo }} branch: ${{ inputs.branch }} timeout: ${{ inputs.timeout }} - role-to-assume: ${{ vars.BENCHMARK_AWS_ROLE_ARN }} - aws-region: ${{ vars.BENCHMARK_AWS_REGION }} diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 70c0df9223..fbea65ef5a 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -73,5 +73,3 @@ jobs: system: ${{ matrix.system }} repo: ${{ github.event.pull_request.head.repo.full_name }} branch: ${{ github.event.pull_request.head.ref }} - role-to-assume: ${{ vars.BENCHMARK_AWS_ROLE_ARN }} - aws-region: ${{ vars.BENCHMARK_AWS_REGION }} From 2c2ad1407dcf11d6995787042cc7fc4dacfe355e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jul 2026 03:03:17 +0000 Subject: [PATCH 3/3] Launch on the whole set of machine types, with a loop instead of a matrix The composite action now takes space-separated lists of systems and machines and launches one EC2 machine per pair, sequentially, in the same job. A failed launch does not stop the rest; the failures are reported at the end. The daily workflow launches the six ClickHouse variants on nine machine types; the PR workflow loses its matrix; the manual workflow accepts lists. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/actions/launch-benchmark/action.yml | 35 ++++++++++++++------- .github/workflows/README.md | 12 ++++--- .github/workflows/benchmark-daily.yml | 34 +++++++++++--------- .github/workflows/benchmark-manual.yml | 14 ++++----- .github/workflows/benchmark-pr.yml | 16 ++++------ 5 files changed, 63 insertions(+), 48 deletions(-) diff --git a/.github/actions/launch-benchmark/action.yml b/.github/actions/launch-benchmark/action.yml index 9ba5234762..92915ac0e6 100644 --- a/.github/actions/launch-benchmark/action.yml +++ b/.github/actions/launch-benchmark/action.yml @@ -1,17 +1,18 @@ name: "Launch benchmark" description: > - Assume the federated AWS IAM role and launch an EC2 machine that runs one - benchmark unattended. The runner only launches: run-benchmark.sh renders - cloud-init.sh.in and calls ec2 run-instances; the machine clones the - repository, runs /benchmark.sh, sends its log to the results sink - and terminates itself. + Assume the federated AWS IAM role and launch EC2 machines that run + benchmarks unattended: one machine for every (machine type, system) pair. + The runner only launches: run-benchmark.sh renders cloud-init.sh.in and + calls ec2 run-instances; each machine clones the repository, runs + /benchmark.sh, sends its log to the results sink and terminates + itself. inputs: - system: - description: "System to benchmark: a directory name, e.g. clickhouse-parquet" + systems: + description: "Systems to benchmark: directory names, space-separated" required: true - machine: - description: "EC2 instance type" + machines: + description: "EC2 instance types, space-separated" default: "c6a.4xlarge" repo: description: "GitHub repository (owner/name) the machine will clone" @@ -39,9 +40,19 @@ runs: - shell: bash env: - system: ${{ inputs.system }} - machine: ${{ inputs.machine }} + systems: ${{ inputs.systems }} + machines: ${{ inputs.machines }} repo: ${{ inputs.repo }} branch: ${{ inputs.branch }} timeout: ${{ inputs.timeout }} - run: ./run-benchmark.sh + run: | + failed="" + for m in $machines; do + for s in $systems; do + machine="$m" system="$s" ./run-benchmark.sh || failed="$failed $s/$m" + done + done + if [ -n "$failed" ]; then + echo "Failed to launch:$failed" + exit 1 + fi diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9ff661c855..5d309fa4be 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -10,8 +10,8 @@ separate process, implemented separately. | Workflow | Trigger | What it launches | |----------|---------|------------------| -| `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants on c6a.4xlarge from main | -| `benchmark-manual.yml` | manual | any system, machine, repository and branch | +| `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | +| `benchmark-manual.yml` | manual | any systems, machines, repository and branch | | `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval | ## Setup @@ -44,6 +44,8 @@ separate process, implemented separately. reviewer approves the pending deployment. 3. Enough on-demand vCPU quota in the region: the daily run launches six - c6a.4xlarge machines at once, 96 vCPUs, on top of whatever is still - running. `run-benchmark.sh` waits and retries while the quota is - exhausted, but only within the job's 55-minute limit. + systems on nine machine types - 3744 vCPUs if everything runs at once, + most of it in the three metal instances (192 vCPUs each) of the six + systems. `run-benchmark.sh` waits and retries while the quota or the + capacity is exhausted, but only within the job's 55-minute limit; what + could not be launched by then is reported as failed. diff --git a/.github/workflows/benchmark-daily.yml b/.github/workflows/benchmark-daily.yml index d27cd79ddb..ba25f94f1e 100644 --- a/.github/workflows/benchmark-daily.yml +++ b/.github/workflows/benchmark-daily.yml @@ -11,23 +11,29 @@ permissions: jobs: launch: - name: ${{ matrix.system }} runs-on: ubuntu-latest - # The AWS session lasts an hour; run-benchmark.sh retries while the vCPU - # quota is exhausted, so cap the job before the credentials expire. + # Machines are launched one by one, and run-benchmark.sh retries while + # the vCPU quota or the capacity is exhausted, so cap the job before the + # hour-long AWS session expires. timeout-minutes: 55 - strategy: - fail-fast: false - matrix: - system: - - clickhouse - - clickhouse-web - - clickhouse-parquet - - clickhouse-parquet-partitioned - - clickhouse-datalake - - clickhouse-datalake-partitioned steps: - uses: actions/checkout@v4 - uses: ./.github/actions/launch-benchmark with: - system: ${{ matrix.system }} + systems: >- + clickhouse + clickhouse-web + clickhouse-parquet + clickhouse-parquet-partitioned + clickhouse-datalake + clickhouse-datalake-partitioned + machines: >- + c6a.4xlarge + c6a.2xlarge + c6a.xlarge + c6a.large + t3a.small + c6a.metal + c8g.4xlarge + c8g.metal-48xl + c7a.metal-48xl diff --git a/.github/workflows/benchmark-manual.yml b/.github/workflows/benchmark-manual.yml index 746a6d58f1..8ca1f4d254 100644 --- a/.github/workflows/benchmark-manual.yml +++ b/.github/workflows/benchmark-manual.yml @@ -1,15 +1,15 @@ name: "Run a benchmark" -run-name: "Benchmark ${{ inputs.system }} on ${{ inputs.machine }} from ${{ inputs.repo }}@${{ inputs.branch }}" +run-name: "Benchmark ${{ inputs.systems }} on ${{ inputs.machines }} from ${{ inputs.repo }}@${{ inputs.branch }}" on: workflow_dispatch: inputs: - system: - description: "System to benchmark (a directory name)" + systems: + description: "Systems to benchmark (directory names, space-separated)" required: true - machine: - description: "EC2 instance type" + machines: + description: "EC2 instance types (space-separated)" default: "c6a.4xlarge" repo: description: "Repository to clone on the machine" @@ -33,8 +33,8 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/launch-benchmark with: - system: ${{ inputs.system }} - machine: ${{ inputs.machine }} + systems: ${{ inputs.systems }} + machines: ${{ inputs.machines }} repo: ${{ inputs.repo }} branch: ${{ inputs.branch }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index fbea65ef5a..738e354270 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -3,7 +3,7 @@ name: "Benchmark pull request" # pull_request_target: the workflow and the launcher scripts are always taken # from the trusted base branch, and the OIDC token is available for PRs from # forks. Code from the PR never runs on the runner - it only runs on the -# disposable benchmark machine, which clones the PR's repository and branch. +# disposable benchmark machines, which clone the PR's repository and branch. on: pull_request_target: @@ -36,24 +36,24 @@ jobs: # directory - except results and documentation - and the directory # contains benchmark.sh at the PR head, so that added systems are # included and deleted ones are not. - systems='[]' + systems="" if [ -n "$HEAD_REPO" ]; then dirs=$(gh api "repos/$BASE_REPO/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename' \ | grep -v -E '(^|/)results/|\.md$' \ | awk -F/ 'NF > 1 { print $1 }' | sort -u || true) for dir in $dirs; do if gh api --silent "repos/$HEAD_REPO/contents/$dir/benchmark.sh?ref=$HEAD_SHA" 2>/dev/null; then - systems=$(jq -c --arg dir "$dir" '. + [$dir]' <<< "$systems") + systems="$systems $dir" fi done fi + systems="${systems# }" echo "Systems to benchmark: $systems" echo "systems=$systems" >> "$GITHUB_OUTPUT" launch: - name: ${{ matrix.system }} needs: detect - if: needs.detect.outputs.systems != '[]' + if: needs.detect.outputs.systems != '' runs-on: ubuntu-latest # Launching machines for PR code costs money and the code is untrusted: # this environment must be configured with required reviewers. @@ -62,14 +62,10 @@ jobs: id-token: write # To assume the federated IAM role. contents: read timeout-minutes: 55 - strategy: - fail-fast: false - matrix: - system: ${{ fromJSON(needs.detect.outputs.systems) }} steps: - uses: actions/checkout@v4 # The base branch: trusted launcher scripts. - uses: ./.github/actions/launch-benchmark with: - system: ${{ matrix.system }} + systems: ${{ needs.detect.outputs.systems }} repo: ${{ github.event.pull_request.head.repo.full_name }} branch: ${{ github.event.pull_request.head.ref }}