Support overriding the clickhouse binary with a PR build

clickhouse-datalake-partitioned/install

@@ -2,7 +2,7 @@
 set -e
 
 if [ ! -x ./clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
 fi
 
 # Use a userspace page cache sized to ~80% of RAM for S3 object reads.

clickhouse-datalake/install

@@ -2,7 +2,7 @@
 set -e
 
 if [ ! -x ./clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
 fi
 
 # Use a userspace page cache sized to ~80% of RAM for S3 object reads.

clickhouse-parquet-partitioned/install

@@ -2,5 +2,5 @@
 set -e
 
 if [ ! -x ./clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
 fi

clickhouse-parquet/install

@@ -2,5 +2,5 @@
 set -e
 
 if [ ! -x ./clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
 fi

clickhouse-trash-schema/install

@@ -2,7 +2,7 @@
 set -e
 
 if [ ! -x /usr/bin/clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
     sudo ./clickhouse install --noninteractive
 fi
 

clickhouse-web/install

@@ -6,7 +6,7 @@ set -e
 # over HTTP from a public ClickHouse-hosted dataset.
 
 if [ ! -x /usr/bin/clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
     sudo ./clickhouse install --noninteractive
 fi
 

clickhouse/install

@@ -2,7 +2,7 @@
 set -e
 
 if [ ! -x /usr/bin/clickhouse ]; then
-    curl https://clickhouse.com/ | sh
+    ../lib/download-clickhouse
     sudo ./clickhouse install --noninteractive
 fi
 

cloud-init.sh.in

@@ -9,6 +9,12 @@
 # once here so every per-system install/start/load/query inherits it.
 export HOME="${HOME:-/root}"
 
+# Optional ClickHouse PR override (see run-benchmark.sh / lib/download-clickhouse).
+# Exported for every system but only consumed by the clickhouse* install
+# scripts; empty for a normal run. Single-quoted; PR numbers are digits so
+# there is nothing to escape.
+export clickhouse_pr='@clickhouse_pr@'
+
 # c6a.4xlarge has 32 GB RAM with no swap. Loading the 75 GB hits.tsv into
 # row-oriented databases (mysql, mariadb, postgres, mongodb, cratedb) and
 # in-process Python servers (pandas/polars/duckdb-dataframe) fills RAM
@@ -43,6 +49,11 @@ jq -r '.tuned' template.json | tee -a log
 echo -n 'Tags: ' | tee -a log
 jq -c -r '.tags' template.json | tee -a log
 
+# Record the ClickHouse PR override (empty on a normal run) so PR-build results
+# can be filtered apart later. sink.parser extracts this into the
+# results.clickhouse_pr column; see prepare-database.sql.
+echo "ClickHouse PR: ${clickhouse_pr}" | tee -a log
+
 echo -n 'Disk usage before: ' | tee -a log
 df -B1 / | tail -n1 | awk '{ print $3 }' | tee -a log
 

lib/download-clickhouse

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Put the `clickhouse` universal binary in the current directory.
+#
+# Default: download the latest stable build from clickhouse.com — exactly what
+# every clickhouse* install script did inline (`curl https://clickhouse.com/ |
+# sh`) before this helper existed.
+#
+# Override: when the `clickhouse_pr` environment variable names a
+# ClickHouse/ClickHouse pull-request number, fetch that PR's CI release build
+# instead of the stable one. This is how an operator benchmarks an unreleased
+# binary straight from a PR:
+#
+#     clickhouse_pr=81944 system=clickhouse ./run-benchmark.sh
+#
+# `clickhouse_pr` is set on the operator side, threaded to the VM through
+# cloud-init, and consumed ONLY here — so only the clickhouse* systems react to
+# it; every other system ignores it.
+#
+# "The binary from the PR" is resolved as: the newest commit in the PR whose
+# release-build CI job for THIS machine's architecture finished successfully,
+# then the raw self-extracting binary that job uploaded to S3. Walking from the
+# newest commit backwards (rather than pinning the tip) means a tip commit whose
+# build is still running, failed, or was skipped (e.g. a docs-only commit) falls
+# back to the last commit that does have a finished build.
+set -euo pipefail
+
+# No PR requested: the default path.
+if [ -z "${clickhouse_pr:-}" ]; then
+    curl https://clickhouse.com/ | sh
+    exit 0
+fi
+
+pr="$clickhouse_pr"
+api="https://api.github.com/repos/ClickHouse/ClickHouse"
+
+# ClickHouse names its release builds by short arch (amd / arm). Map this
+# machine's architecture onto that; reject targets that have no release build.
+case "$(uname -m)" in
+    x86_64 | amd64)  arch=amd ;;
+    aarch64 | arm64) arch=arm ;;
+    *)
+        echo "clickhouse_pr: no release build for architecture '$(uname -m)'" >&2
+        exit 1
+        ;;
+esac
+check_name="Build (${arch}_release)"   # CI check-run name for the release build
+build_dir="build_${arch}_release"      # S3 folder that build uploads into
+
+# GitHub's anonymous API limit is 60 requests/hour; if the operator exported a
+# GITHUB_TOKEN, use it to lift the limit (helps when many machines launch at
+# once and each makes a handful of calls).
+gh() {
+    if [ -n "${GITHUB_TOKEN:-}" ]; then
+        curl -fsSL -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            -H "Accept: application/vnd.github+json" "$@"
+    else
+        curl -fsSL -H "Accept: application/vnd.github+json" "$@"
+    fi
+}
+
+# Candidate commits, newest first: the PR head (the usual answer) followed by
+# the rest of the PR's commits newest->oldest as a fallback. The PR-commits
+# endpoint returns oldest->newest, so `tac` flips it; awk drops blanks and the
+# duplicate head. (For a PR with >100 commits only the first page's commits act
+# as fallback, but the head — checked first — covers the normal case.)
+head_sha=$(gh "${api}/pulls/${pr}" | jq -r '.head.sha')
+rest=$(gh "${api}/pulls/${pr}/commits?per_page=100" | jq -r '.[].sha' | tac)
+candidates=$(printf '%s\n%s\n' "$head_sha" "$rest" | awk 'NF && !seen[$0]++')
+
+echo "clickhouse_pr=${pr}: looking for a successful '${check_name}' build..." >&2
+sha=""
+while IFS= read -r commit; do
+    [ -z "$commit" ] && continue
+    # Filter server-side by check name so we don't have to page through the
+    # ~170 check-runs a ClickHouse commit accumulates.
+    if gh -G --data-urlencode "check_name=${check_name}" \
+            "${api}/commits/${commit}/check-runs" \
+            | jq -e 'any(.check_runs[]; .conclusion == "success")' >/dev/null; then
+        sha="$commit"
+        break
+    fi
+done <<< "$candidates"
+
+if [ -z "$sha" ]; then
+    echo "clickhouse_pr=${pr}: found no commit with a successful '${check_name}'." >&2
+    echo "clickhouse_pr=${pr}: is CI still running, or did the release build fail?" >&2
+    exit 1
+fi
+
+url="https://clickhouse-builds.s3.amazonaws.com/PRs/${pr}/${sha}/${build_dir}/clickhouse"
+echo "clickhouse_pr=${pr}: commit ${sha}" >&2
+echo "clickhouse_pr=${pr}: downloading ${url}" >&2
+wget --continue --progress=dot:giga "$url" -O clickhouse
+chmod +x clickhouse

prepare-database.sql

@@ -17,6 +17,9 @@ CREATE TABLE sink.results
     proprietary String,
     tuned String,
     tags String,
+    -- ClickHouse/ClickHouse PR number when the run benchmarked a PR build
+    -- (clickhouse_pr override, clickhouse* systems only); empty otherwise.
+    clickhouse_pr String,
     total_time UInt64,
     disk_space_diff UInt64,
     load_time Float64,
@@ -43,6 +46,7 @@ WITH
     extract(content, 'Proprietary: ([^\n]+)') AS proprietary,
     extract(content, 'Tuned: ([^\n]+)') AS tuned,
     extract(content, 'Tags: ([^\n]+)') AS tags,
+    extract(content, 'ClickHouse PR: ([^\n]+)') AS clickhouse_pr,
 
     toUInt64OrZero(extract(content, 'Disk usage after: (\d+)')) - toUInt64OrZero(extract(content, 'Disk usage before: (\d+)')) AS disk_space_diff,
     toUInt64OrZero(extract(content, 'Total time: (\d+)')) AS total_time,
@@ -65,7 +69,7 @@ WITH
     load_time IS NOT NULL AND length(runtimes) = 43 AND data_size >= 5000000000
         AND arrayExists(x -> arrayExists(y -> toFloat64OrZero(y) > 0.1, x), runtimes) AS good
 
-SELECT time, system, machine, system_name, proprietary, tuned, tags, total_time, disk_space_diff, load_time, data_size, length(runtimes) AS num_results, runtimes, runtimes_formatted,
+SELECT time, system, machine, system_name, proprietary, tuned, tags, clickhouse_pr, total_time, disk_space_diff, load_time, data_size, length(runtimes) AS num_results, runtimes, runtimes_formatted,
     concurrent_qps, concurrent_error_ratio,
 '{
     "system": "' || system_name || '",

run-benchmark.sh

@@ -5,19 +5,26 @@ system=${system:=clickhouse}
 repo=${repo:=ClickHouse/ClickBench}
 branch=${branch:=main}
 
+# Optional ClickHouse/ClickHouse PR number. When set, the clickhouse* systems
+# benchmark that PR's CI release build instead of the latest stable one; every
+# other system ignores it. Threaded to the VM below and resolved there by
+# lib/download-clickhouse.
+clickhouse_pr=${clickhouse_pr:-}
+
 arch=$(aws ec2 describe-instance-types --instance-types $machine --query 'InstanceTypes[0].ProcessorInfo.SupportedArchitectures' --output text)
 ami=$(aws ec2 describe-images --owners amazon --filters "Name=name,Values=ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04*" "Name=architecture,Values=${arch}" "Name=state,Values=available" --query 'sort_by(Images, &CreationDate) | [-1].[ImageId]' --output text)
 
 # Global per-system benchmark timeout — substituted at render time.
 # Default keeps the 10h cap that worked for the slowest OLTP systems.
 timeout="${timeout:-36000}"
 
-awk -v sys="$system" -v repo="$repo" -v branch="$branch" -v t="$timeout" '
+awk -v sys="$system" -v repo="$repo" -v branch="$branch" -v t="$timeout" -v chpr="$clickhouse_pr" '
 {
     gsub(/@system@/, sys)
     gsub(/@repo@/, repo)
     gsub(/@branch@/, branch)
     gsub(/@timeout@/, t)
+    gsub(/@clickhouse_pr@/, chpr)
     print
 }' cloud-init.sh.in > cloud-init.sh