diff --git a/.github/actions/launch-benchmark/action.yml b/.github/actions/launch-benchmark/action.yml new file mode 100644 index 0000000000..92915ac0e6 --- /dev/null +++ b/.github/actions/launch-benchmark/action.yml @@ -0,0 +1,58 @@ +name: "Launch benchmark" +description: > + Assume the federated AWS IAM role and launch EC2 machines that run + benchmarks unattended: one machine for every (machine type, system) pair. + The runner only launches: run-benchmark.sh renders cloud-init.sh.in and + calls ec2 run-instances; each machine clones the repository, runs + /benchmark.sh, sends its log to the results sink and terminates + itself. + +inputs: + systems: + description: "Systems to benchmark: directory names, space-separated" + required: true + machines: + description: "EC2 instance types, space-separated" + default: "c6a.4xlarge" + repo: + description: "GitHub repository (owner/name) the machine will clone" + default: "ClickHouse/ClickBench" + branch: + description: "Branch of that repository to clone" + default: "main" + timeout: + description: "Time limit for benchmark.sh on the machine, seconds" + default: "36000" + aws-region: + description: "AWS region to launch the machine in" + default: "us-east-1" + +runs: + using: "composite" + steps: + # The role's trust policy only admits workflow runs of this repository, + # so assuming it fails everywhere else, including forks. + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::264113785604:role/ClickBenchLauncher + aws-region: ${{ inputs.aws-region }} + role-session-name: clickbench-${{ github.run_id }} + + - shell: bash + env: + systems: ${{ inputs.systems }} + machines: ${{ inputs.machines }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + timeout: ${{ inputs.timeout }} + run: | + failed="" + for m in $machines; do + for s in $systems; do + machine="$m" system="$s" ./run-benchmark.sh || failed="$failed $s/$m" + done + done + if [ -n "$failed" ]; then + echo "Failed to launch:$failed" + exit 1 + fi diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..5d309fa4be --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,51 @@ +# Benchmark automation + +These workflows launch benchmark machines on AWS. The GitHub-hosted runner is +used only to launch: it assumes a federated IAM role and runs +`run-benchmark.sh`, which starts a self-terminating EC2 machine. The machine +clones the repository, runs `/benchmark.sh` under cloud-init (see +`cloud-init.sh.in`), sends its log to the results sink at play.clickhouse.com, +and shuts down. Collecting the results back into `/results/` is a +separate process, implemented separately. + +| Workflow | Trigger | What it launches | +|----------|---------|------------------| +| `benchmark-daily.yml` | daily, 02:00 UTC | the ClickHouse variants, each on the whole set of machine types, from main | +| `benchmark-manual.yml` | manual | any systems, machines, repository and branch | +| `benchmark-pr.yml` | pull requests | the systems whose directories the PR changes (results and *.md files don't count), from the PR's repository and branch, after manual approval | + +## Setup + +1. An IAM role for GitHub's OIDC provider, restricted to this repository: + + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { "Federated": "arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com" }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" }, + "StringLike": { "token.actions.githubusercontent.com:sub": "repo:ClickHouse/ClickBench:*" } + } + }] + } + ``` + + The permissions policy needs `ec2:RunInstances`, `ec2:CreateTags`, + `ec2:DescribeImages`, and `ec2:DescribeInstanceTypes`. + + The role's ARN and the region (us-east-1) are set in + `.github/actions/launch-benchmark/action.yml`. + +2. An environment named `benchmark-approval` with required reviewers. It + gates the PR workflow: nothing is launched for a pull request until a + reviewer approves the pending deployment. + +3. Enough on-demand vCPU quota in the region: the daily run launches six + systems on nine machine types - 3744 vCPUs if everything runs at once, + most of it in the three metal instances (192 vCPUs each) of the six + systems. `run-benchmark.sh` waits and retries while the quota or the + capacity is exhausted, but only within the job's 55-minute limit; what + could not be launched by then is reported as failed. diff --git a/.github/workflows/benchmark-daily.yml b/.github/workflows/benchmark-daily.yml new file mode 100644 index 0000000000..ba25f94f1e --- /dev/null +++ b/.github/workflows/benchmark-daily.yml @@ -0,0 +1,39 @@ +name: "Daily benchmarks: ClickHouse" + +on: + workflow_dispatch: # This allows manual trigger from the UI + schedule: + - cron: '0 2 * * *' + +permissions: + id-token: write # To assume the federated IAM role. + contents: read + +jobs: + launch: + runs-on: ubuntu-latest + # Machines are launched one by one, and run-benchmark.sh retries while + # the vCPU quota or the capacity is exhausted, so cap the job before the + # hour-long AWS session expires. + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/launch-benchmark + with: + systems: >- + clickhouse + clickhouse-web + clickhouse-parquet + clickhouse-parquet-partitioned + clickhouse-datalake + clickhouse-datalake-partitioned + machines: >- + c6a.4xlarge + c6a.2xlarge + c6a.xlarge + c6a.large + t3a.small + c6a.metal + c8g.4xlarge + c8g.metal-48xl + c7a.metal-48xl diff --git a/.github/workflows/benchmark-manual.yml b/.github/workflows/benchmark-manual.yml new file mode 100644 index 0000000000..8ca1f4d254 --- /dev/null +++ b/.github/workflows/benchmark-manual.yml @@ -0,0 +1,40 @@ +name: "Run a benchmark" + +run-name: "Benchmark ${{ inputs.systems }} on ${{ inputs.machines }} from ${{ inputs.repo }}@${{ inputs.branch }}" + +on: + workflow_dispatch: + inputs: + systems: + description: "Systems to benchmark (directory names, space-separated)" + required: true + machines: + description: "EC2 instance types (space-separated)" + default: "c6a.4xlarge" + repo: + description: "Repository to clone on the machine" + default: "ClickHouse/ClickBench" + branch: + description: "Branch to clone" + default: "main" + timeout: + description: "Time limit for benchmark.sh, seconds" + default: "36000" + +permissions: + id-token: write # To assume the federated IAM role. + contents: read + +jobs: + launch: + runs-on: ubuntu-latest + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/launch-benchmark + with: + systems: ${{ inputs.systems }} + machines: ${{ inputs.machines }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml new file mode 100644 index 0000000000..738e354270 --- /dev/null +++ b/.github/workflows/benchmark-pr.yml @@ -0,0 +1,71 @@ +name: "Benchmark pull request" + +# pull_request_target: the workflow and the launcher scripts are always taken +# from the trusted base branch, and the OIDC token is available for PRs from +# forks. Code from the PR never runs on the runner - it only runs on the +# disposable benchmark machines, which clone the PR's repository and branch. +on: + pull_request_target: + +permissions: + contents: read + +# A new push supersedes queued runs, including ones still waiting for approval. +concurrency: + group: benchmark-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + detect: + name: Detect changed systems + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + systems: ${{ steps.changed.outputs.systems }} + steps: + - id: changed + env: + GH_TOKEN: ${{ github.token }} + BASE_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + # A system should be benchmarked if the PR touches files in its + # directory - except results and documentation - and the directory + # contains benchmark.sh at the PR head, so that added systems are + # included and deleted ones are not. + systems="" + if [ -n "$HEAD_REPO" ]; then + dirs=$(gh api "repos/$BASE_REPO/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename' \ + | grep -v -E '(^|/)results/|\.md$' \ + | awk -F/ 'NF > 1 { print $1 }' | sort -u || true) + for dir in $dirs; do + if gh api --silent "repos/$HEAD_REPO/contents/$dir/benchmark.sh?ref=$HEAD_SHA" 2>/dev/null; then + systems="$systems $dir" + fi + done + fi + systems="${systems# }" + echo "Systems to benchmark: $systems" + echo "systems=$systems" >> "$GITHUB_OUTPUT" + + launch: + needs: detect + if: needs.detect.outputs.systems != '' + runs-on: ubuntu-latest + # Launching machines for PR code costs money and the code is untrusted: + # this environment must be configured with required reviewers. + environment: benchmark-approval + permissions: + id-token: write # To assume the federated IAM role. + contents: read + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 # The base branch: trusted launcher scripts. + - uses: ./.github/actions/launch-benchmark + with: + systems: ${{ needs.detect.outputs.systems }} + repo: ${{ github.event.pull_request.head.repo.full_name }} + branch: ${{ github.event.pull_request.head.ref }}