From 5bd453d5ce961cfc3d33bd1503850333bfc322dd Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 11 Aug 2026 18:18:53 +0300 Subject: [PATCH 1/2] Add persistent Benchee benchmarks --- .formatter.exs | 2 +- .github/workflows/benchmark.yml | 124 ++++++++++++++++++++++++++++++++ AGENTS.md | 83 +++++++++++++++++++++ bench/benchmark.exs | 80 +++++++++++++++++++++ bench/compare.sql | 44 ++++++++++++ bench/json_formatter.exs | 100 ++++++++++++++++++++++++++ mix.exs | 7 ++ mix.lock | 3 + 8 files changed, 442 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 AGENTS.md create mode 100644 bench/benchmark.exs create mode 100644 bench/compare.sql create mode 100644 bench/json_formatter.exs diff --git a/.formatter.exs b/.formatter.exs index 17f7637c..dc006466 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,5 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{lib,test}/**/*.{ex,exs}"], + inputs: ["{mix,.formatter}.exs", "{lib,test,bench}/**/*.{ex,exs}"], import_deps: [:stream_data] ] diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..a107216b --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,124 @@ +name: benchmark + +on: + pull_request: + push: + branches: [master] + schedule: + - cron: "17 10 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: benchmark-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + row-binary: + runs-on: ubuntu-latest + env: + MIX_ENV: dev + BENCHMARK_MACHINE_PREFIX: github-actions + BENCHMARK_VERSION: ${{ github.event.pull_request.head.sha || github.sha }} + BENCHMARK_TIME: "3.0" + BENCHMARK_WARMUP: "1.0" + BENCHMARK_PROFILE: tprof + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - id: beam + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.19" + otp-version: "28" + + - uses: actions/cache@v6 + with: + path: | + deps + _build + key: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('mix.lock') }} + restore-keys: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}- + + - run: mix deps.get --only dev + - run: mix compile --warnings-as-errors + - run: mix benchmark + + - name: Locate benchmark result + run: | + result_path=$(cat bench/output/current-result-path) + machine=$(jq --raw-output .machine "$result_path") + echo "BENCHMARK_RESULT_PATH=$result_path" >> "$GITHUB_ENV" + echo "BENCHMARK_MACHINE=$machine" >> "$GITHUB_ENV" + + - name: Install ClickHouse Local + env: + DO_NOT_TRACK: "1" + run: | + curl --fail --silent --show-error https://clickhouse.com/cli | sh + "$HOME/.local/bin/clickhousectl" local use latest + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Load benchmark history + run: | + if git fetch origin benchmark-results; then + git worktree add bench-history FETCH_HEAD + else + git worktree add --detach bench-history + git -C bench-history switch --orphan benchmark-results + fi + + mkdir -p bench/compare + cp "$BENCHMARK_RESULT_PATH" bench/compare/current.json + + if test -f "bench-history/latest/$BENCHMARK_MACHINE/benchmark.json"; then + cp "bench-history/latest/$BENCHMARK_MACHINE/benchmark.json" bench/compare/baseline.json + else + cp bench/compare/current.json bench/compare/baseline.json + echo "BENCHMARK_BASELINE_NOTE=No prior baseline exists; this run establishes it." >> "$GITHUB_ENV" + fi + + - name: Compare with latest master benchmark + run: | + clickhouse local --queries-file bench/compare.sql > bench/compare/comment.md + { + echo "## RowBinary benchmark" + echo + echo "${BENCHMARK_BASELINE_NOTE:-Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise.}" + echo + cat bench/compare/comment.md + echo + echo "Commit: \`$BENCHMARK_VERSION\` · Machine: \`$BENCHMARK_MACHINE\` · Profiler: TProf · Full JSON results are attached to this workflow run." + } > bench/compare/pr-comment.md + cat bench/compare/pr-comment.md + + - uses: actions/upload-artifact@v7 + with: + name: row-binary-benchmark-${{ env.BENCHMARK_VERSION }} + path: bench/output/machine=*/version=${{ env.BENCHMARK_VERSION }}/benchmark.json + + - name: Comment on pull request + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: gh pr comment "${{ github.event.pull_request.number }}" --edit-last --create-if-none --body-file bench/compare/pr-comment.md + + - name: Store master result + if: github.event_name == 'push' || github.event_name == 'schedule' + run: | + destination="data/machine=$BENCHMARK_MACHINE/version=$BENCHMARK_VERSION" + mkdir -p "bench-history/$destination" "bench-history/latest/$BENCHMARK_MACHINE" + cp -R "bench/output/machine=$BENCHMARK_MACHINE/version=$BENCHMARK_VERSION/." "bench-history/$destination/" + cp "bench-history/$destination/benchmark.json" "bench-history/latest/$BENCHMARK_MACHINE/benchmark.json" + git -C bench-history add data latest + git -C bench-history config user.name github-actions[bot] + git -C bench-history config user.email 41898282+github-actions[bot]@users.noreply.github.com + git -C bench-history commit -m "Benchmark $BENCHMARK_VERSION" + git -C bench-history push origin HEAD:benchmark-results diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..528c1b5a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,83 @@ +# Benchmark history + +Performance results live on the orphan `benchmark-results` branch. They are stored as self-describing Benchee JSON under: + +```text +data/machine=/version=/benchmark.json +latest//benchmark.json +``` + +Machine partitions are derived from the execution environment, OS, CPU model, online core count, total memory, and architecture. CI partitions start with `github-actions-`, while local partitions start with `local-`. Never compare results across machine partitions. A benchmark version is the commit SHA whose code was measured. + +The queries below use `clickhouse local`. If the ClickHouse binary is not installed, install it with `curl https://clickhouse.com/cli | sh` followed by `~/.local/bin/clickhousectl local use latest`. + +Run benchmarks locally with `mix benchmark`. To include Benchee's built-in TProf pass, use `BENCHMARK_PROFILE=tprof mix benchmark`. JSON results are written below `bench/output/`, partitioned by detected machine specifications and SHA. Each file includes schema/run identity; timestamp and CI provenance; OS, architecture, CPU, cores, memory, Elixir, Erlang, and JIT details; benchmark configuration and units; and raw samples plus runtime, memory, and reduction statistics for every scenario. Override `BENCHMARK_MACHINE_PREFIX`, `BENCHMARK_VERSION`, `BENCHMARK_TIME`, or `BENCHMARK_WARMUP` when a reproducible series needs fixed settings; hardware fields are always detected. + +An abridged result from the demo RowBinary benchmark looks like this. Actual files contain all samples and statistics, plus similarly shaped `memory_usage` and `reductions` collections for each scenario. + +```json +{ + "schema_version": 1, + "benchmark_version": "e19815a...", + "machine": "local-macOS-Apple-M2-8-cores-8-GB-aarch64-apple-darwin", + "generated_at": "2026-08-03T12:43:41Z", + "ci": false, + "units": { + "configuration_time": "nanosecond", + "run_time_samples": "nanosecond", + "run_time_ips": "iterations_per_second", + "memory_usage_samples": "byte", + "reductions_samples": "count" + }, + "system": { + "os": "macOS", + "architecture": "aarch64-apple-darwin", + "cpu_speed": "Apple M2", + "num_cores": 8, + "available_memory": "8 GB", + "elixir": "1.20.2", + "erlang": "29.0.3", + "jit_enabled?": true + }, + "configuration": { + "time": 20000000.0, + "warmup": 10000000.0, + "parallel": 1, + "percentiles": [50, 99], + "max_sample_size": 20000 + }, + "scenarios": [ + { + "job_name": "RowBinary.encode_rows/2", + "input_name": "1000 rows", + "run_time": { + "samples": [357000, 339500, 551708], + "statistics": { + "ips": 2485.05, + "average": 402405.88, + "median": 388562.5, + "std_dev_ratio": 0.1445, + "percentiles": {"25": 349646.0, "50": 388562.5, "75": 454552.5, "99": 555458.0}, + "sample_size": 50 + } + } + } + ] +} +``` + +To query history without modifying the working tree: + +```sh +git fetch origin benchmark-results +git worktree add .context/benchmark-results origin/benchmark-results +clickhouse local -q "SELECT _file, scenario.job_name, scenario.input_name, round(scenario.run_time.statistics.ips, 2) AS ips FROM file('.context/benchmark-results/data/machine=github-actions-*/version=*/benchmark.json', JSONEachRow) ARRAY JOIN scenarios AS scenario ORDER BY _file DESC, scenario.job_name, scenario.input_name FORMAT PrettyCompact" +``` + +For the same comparison used by CI, copy two result files to `bench/compare/baseline.json` and `bench/compare/current.json`, then run: + +```sh +clickhouse local --queries-file bench/compare.sql +``` + +The SQL treats an absolute throughput change below 5% as noise (`no material change`). Inspect raw samples in `scenario.run_time.samples` before attributing small changes to code. Remove the `.context/benchmark-results` worktree with `git worktree remove .context/benchmark-results` when finished. diff --git a/bench/benchmark.exs b/bench/benchmark.exs new file mode 100644 index 00000000..0f5bf39a --- /dev/null +++ b/bench/benchmark.exs @@ -0,0 +1,80 @@ +alias Ch.RowBinary + +Code.require_file("json_formatter.exs", __DIR__) + +types = ["UInt64", "String", "Array(UInt8)", "DateTime64(3, 'UTC')", "DateTime"] +titles = ["Golang SQL database driver", "Phoenix app event", "billing webhook payload"] +bytes = Enum.to_list(1..16) +base_datetime = DateTime.from_naive!(~N[2026-01-01 00:00:00.000], "Etc/UTC") +base_naive = ~N[2026-01-01 00:00:00] + +make_rows = fn count -> + Enum.map(1..count, fn i -> + [ + i, + Enum.at(titles, rem(i, length(titles))), + Enum.take(bytes, rem(i, length(bytes)) + 1), + DateTime.add(base_datetime, i * 17, :millisecond), + NaiveDateTime.add(base_naive, i, :second) + ] + end) +end + +inputs = + for count <- [1_000, 10_000], into: %{} do + rows = make_rows.(count) + + {"#{count} rows", + %{rows: rows, encoded: rows |> RowBinary.encode_rows(types) |> IO.iodata_to_binary()}} + end + +version = + System.get_env("BENCHMARK_VERSION") || + case System.cmd("git", ["rev-parse", "HEAD"], stderr_to_stdout: true) do + {sha, 0} -> String.trim(sha) + _ -> "working-tree" + end + +output_root = System.get_env("BENCHMARK_OUTPUT_DIR", "bench/output") + +seconds = fn name, default -> + case System.get_env(name) do + nil -> default + value -> String.to_float(value) + end +end + +profile_after = + case System.get_env("BENCHMARK_PROFILE") do + nil -> false + "" -> false + "false" -> false + profiler -> String.to_existing_atom(profiler) + end + +Benchee.run( + %{ + "RowBinary.encode_rows/2" => fn %{rows: rows} -> RowBinary.encode_rows(rows, types) end, + "RowBinary.decode_rows/2" => fn %{encoded: encoded} -> + RowBinary.decode_rows(encoded, types) + end + }, + inputs: inputs, + time: seconds.("BENCHMARK_TIME", 5.0), + warmup: seconds.("BENCHMARK_WARMUP", 2.0), + max_sample_size: 20_000, + measure_function_call_overhead: true, + pre_check: true, + profile_after: profile_after, + formatters: [ + {Benchee.Formatters.Console, extended_statistics: true}, + {Ch.Bench.JSONFormatter, + output_root: output_root, + benchmark_version: version, + machine_prefix: + System.get_env( + "BENCHMARK_MACHINE_PREFIX", + if(System.get_env("CI") == "true", do: "ci", else: "local") + )} + ] +) diff --git a/bench/compare.sql b/bench/compare.sql new file mode 100644 index 00000000..f0d75cf1 --- /dev/null +++ b/bench/compare.sql @@ -0,0 +1,44 @@ +SET join_use_nulls = 1; + +WITH +baseline AS ( + SELECT + scenario.job_name AS job_name, + scenario.input_name AS input_name, + scenario.run_time.statistics.ips AS baseline_ips, + scenario.run_time.statistics.median AS baseline_median_ns + FROM file('bench/compare/baseline.json', JSONEachRow) + ARRAY JOIN scenarios AS scenario +), +current AS ( + SELECT + scenario.job_name AS job_name, + scenario.input_name AS input_name, + scenario.run_time.statistics.ips AS current_ips, + scenario.run_time.statistics.median AS current_median_ns + FROM file('bench/compare/current.json', JSONEachRow) + ARRAY JOIN scenarios AS scenario +) +SELECT + coalesce(current.job_name, baseline.job_name) AS benchmark, + coalesce(current.input_name, baseline.input_name) AS input, + round(baseline_ips, 2) AS baseline_ips, + round(current_ips, 2) AS current_ips, + multiIf( + baseline_ips IS NULL, 'new', + current_ips IS NULL, 'removed', + abs((current_ips / baseline_ips - 1) * 100) < 5, 'no material change', + current_ips > baseline_ips, 'faster', + 'slower' + ) AS result, + if( + baseline_ips IS NULL OR current_ips IS NULL, + NULL, + round((current_ips / baseline_ips - 1) * 100, 2) + ) AS change_percent, + round(baseline_median_ns / 1000000, 3) AS baseline_median_ms, + round(current_median_ns / 1000000, 3) AS current_median_ms +FROM baseline +FULL OUTER JOIN current USING (job_name, input_name) +ORDER BY benchmark, input +FORMAT Markdown; diff --git a/bench/json_formatter.exs b/bench/json_formatter.exs new file mode 100644 index 00000000..c3dbc50b --- /dev/null +++ b/bench/json_formatter.exs @@ -0,0 +1,100 @@ +defmodule Ch.Bench.JSONFormatter do + @behaviour Benchee.Formatter + + @configuration_fields [ + :time, + :warmup, + :memory_time, + :reduction_time, + :parallel, + :percentiles, + :max_sample_size, + :exclude_outliers, + :measure_function_call_overhead + ] + + @impl Benchee.Formatter + def format(%Benchee.Suite{} = suite, opts) do + version = Map.fetch!(opts, :benchmark_version) + + system = + suite.system + |> Map.from_struct() + |> Map.put(:architecture, :erlang.system_info(:system_architecture) |> List.to_string()) + + machine = machine_key(Map.fetch!(opts, :machine_prefix), system) + output_root = Map.fetch!(opts, :output_root) + + path = + Path.join([output_root, "machine=#{machine}", "version=#{safe(version)}", "benchmark.json"]) + + data = + %{ + schema_version: 1, + benchmark_version: version, + machine: machine, + generated_at: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601(), + ci: System.get_env("CI") == "true", + units: %{ + configuration_time: "nanosecond", + run_time_samples: "nanosecond", + run_time_ips: "iterations_per_second", + memory_usage_samples: "byte", + reductions_samples: "count" + }, + system: system, + configuration: Map.take(Map.from_struct(suite.configuration), @configuration_fields), + scenarios: Enum.map(suite.scenarios, &scenario/1) + } + |> json_safe() + |> JSON.encode_to_iodata!() + + %{data: data, path: path, output_root: output_root} + end + + @impl Benchee.Formatter + def write(%{data: data, path: path, output_root: output_root}, _opts) do + path |> Path.dirname() |> File.mkdir_p!() + File.write!(path, data) + File.write!(Path.join(output_root, "current-result-path"), path) + IO.puts("Benchmark artifact: #{path}") + end + + defp machine_key(prefix, system) do + [ + prefix, + system.os, + system.cpu_speed, + "#{system.num_cores}-cores", + system.available_memory, + system.architecture + ] + |> Enum.join("-") + |> safe() + end + + defp safe(value), do: String.replace(value, ~r/[^a-zA-Z0-9._-]/, "-") + + defp scenario(scenario) do + %{ + name: scenario.name, + job_name: scenario.job_name, + input_name: scenario.input_name, + tag: scenario.tag, + run_time: scenario.run_time_data, + memory_usage: scenario.memory_usage_data, + reductions: scenario.reductions_data + } + end + + defp json_safe(%_{} = struct), do: struct |> Map.from_struct() |> json_safe() + + defp json_safe(map) when is_map(map) do + Map.new(map, fn {key, value} -> {to_string(key), json_safe(value)} end) + end + + defp json_safe(list) when is_list(list), do: Enum.map(list, &json_safe/1) + defp json_safe(tuple) when is_tuple(tuple), do: tuple |> Tuple.to_list() |> json_safe() + defp json_safe(atom) when is_atom(atom) and atom not in [true, false, nil], do: to_string(atom) + defp json_safe(value), do: value +end diff --git a/mix.exs b/mix.exs index da893160..040f5c63 100644 --- a/mix.exs +++ b/mix.exs @@ -13,6 +13,7 @@ defmodule Ch.MixProject do elixir: "~> 1.18", elixirc_paths: elixirc_paths(Mix.env()), deps: deps(), + aliases: aliases(), name: "Ch", description: "HTTP ClickHouse client for Elixir", docs: docs(), @@ -31,6 +32,7 @@ defmodule Ch.MixProject do def cli do [ preferred_envs: [ + benchmark: :dev, coveralls: :test, "coveralls.detail": :test, "coveralls.json": :test, @@ -52,6 +54,7 @@ defmodule Ch.MixProject do {:nimble_options, "~> 1.1"}, {:decimal, "~> 3.0"}, {:ecto, "~> 3.14.0", optional: true}, + {:benchee, "~> 1.5", only: :dev}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev}, {:tz, "~> 0.28.1", only: [:dev, :test]}, @@ -60,6 +63,10 @@ defmodule Ch.MixProject do ] end + defp aliases do + [benchmark: "run bench/benchmark.exs"] + end + defp docs do [ source_url: @source_url, diff --git a/mix.lock b/mix.lock index 000c26ae..396c76b6 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,7 @@ %{ + "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"}, "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, + "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "ecto": {:hex, :ecto, "3.14.1", "7b740d87bdf45996aa0c2c2e081640906f10caa7ce5ba328fd294c7d49d0cc6f", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "24b991956796700f467d0a3ef3d303138a3ef9ddddf8b98f43758ee067b20a30"}, @@ -15,6 +17,7 @@ "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "tz": {:hex, :tz, "0.28.2", "6c47f3d1a8ee5c33a1d8f0ba49e5a851b0a30c408a587d907aff6e71228b3b32", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:mint, "~> 1.6", [hex: :mint, repo: "hexpm", optional: true]}], "hexpm", "a6bf7355a33f0a7511602ab4566432ac0901d8abff81e94e455bca19708a0c87"}, From 2fb76c449dff20bd8274e9c7312ce119d2d1e3bb Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 13 Aug 2026 11:22:51 +0300 Subject: [PATCH 2/2] Remove benchmark environment overrides --- .github/workflows/benchmark.yml | 53 +++++++++++++++++++-------------- AGENTS.md | 2 +- bench/benchmark.exs | 37 ++++++----------------- 3 files changed, 41 insertions(+), 51 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a107216b..17f5681d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,13 +19,6 @@ concurrency: jobs: row-binary: runs-on: ubuntu-latest - env: - MIX_ENV: dev - BENCHMARK_MACHINE_PREFIX: github-actions - BENCHMARK_VERSION: ${{ github.event.pull_request.head.sha || github.sha }} - BENCHMARK_TIME: "3.0" - BENCHMARK_WARMUP: "1.0" - BENCHMARK_PROFILE: tprof steps: - uses: actions/checkout@v7 @@ -52,11 +45,14 @@ jobs: - run: mix benchmark - name: Locate benchmark result + id: benchmark-result run: | result_path=$(cat bench/output/current-result-path) machine=$(jq --raw-output .machine "$result_path") - echo "BENCHMARK_RESULT_PATH=$result_path" >> "$GITHUB_ENV" - echo "BENCHMARK_MACHINE=$machine" >> "$GITHUB_ENV" + version=$(jq --raw-output .benchmark_version "$result_path") + echo "path=$result_path" >> "$GITHUB_OUTPUT" + echo "machine=$machine" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" - name: Install ClickHouse Local env: @@ -67,7 +63,11 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Load benchmark history + id: benchmark-history run: | + result_path="${{ steps.benchmark-result.outputs.path }}" + machine="${{ steps.benchmark-result.outputs.machine }}" + if git fetch origin benchmark-results; then git worktree add bench-history FETCH_HEAD else @@ -76,33 +76,39 @@ jobs: fi mkdir -p bench/compare - cp "$BENCHMARK_RESULT_PATH" bench/compare/current.json + cp "$result_path" bench/compare/current.json - if test -f "bench-history/latest/$BENCHMARK_MACHINE/benchmark.json"; then - cp "bench-history/latest/$BENCHMARK_MACHINE/benchmark.json" bench/compare/baseline.json + if test -f "bench-history/latest/$machine/benchmark.json"; then + cp "bench-history/latest/$machine/benchmark.json" bench/compare/baseline.json else cp bench/compare/current.json bench/compare/baseline.json - echo "BENCHMARK_BASELINE_NOTE=No prior baseline exists; this run establishes it." >> "$GITHUB_ENV" + echo "note=No prior baseline exists; this run establishes it." >> "$GITHUB_OUTPUT" fi - name: Compare with latest master benchmark run: | + version="${{ steps.benchmark-result.outputs.version }}" + machine="${{ steps.benchmark-result.outputs.machine }}" + baseline_note="${{ steps.benchmark-history.outputs.note }}" + if test -z "$baseline_note"; then + baseline_note="Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise." + fi clickhouse local --queries-file bench/compare.sql > bench/compare/comment.md { echo "## RowBinary benchmark" echo - echo "${BENCHMARK_BASELINE_NOTE:-Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise.}" + echo "$baseline_note" echo cat bench/compare/comment.md echo - echo "Commit: \`$BENCHMARK_VERSION\` · Machine: \`$BENCHMARK_MACHINE\` · Profiler: TProf · Full JSON results are attached to this workflow run." + echo "Commit: \`$version\` · Machine: \`$machine\` · Profiler: TProf · Full JSON results are attached to this workflow run." } > bench/compare/pr-comment.md cat bench/compare/pr-comment.md - uses: actions/upload-artifact@v7 with: - name: row-binary-benchmark-${{ env.BENCHMARK_VERSION }} - path: bench/output/machine=*/version=${{ env.BENCHMARK_VERSION }}/benchmark.json + name: row-binary-benchmark-${{ steps.benchmark-result.outputs.version }} + path: ${{ steps.benchmark-result.outputs.path }} - name: Comment on pull request if: github.event_name == 'pull_request' @@ -113,12 +119,15 @@ jobs: - name: Store master result if: github.event_name == 'push' || github.event_name == 'schedule' run: | - destination="data/machine=$BENCHMARK_MACHINE/version=$BENCHMARK_VERSION" - mkdir -p "bench-history/$destination" "bench-history/latest/$BENCHMARK_MACHINE" - cp -R "bench/output/machine=$BENCHMARK_MACHINE/version=$BENCHMARK_VERSION/." "bench-history/$destination/" - cp "bench-history/$destination/benchmark.json" "bench-history/latest/$BENCHMARK_MACHINE/benchmark.json" + result_path="${{ steps.benchmark-result.outputs.path }}" + machine="${{ steps.benchmark-result.outputs.machine }}" + version="${{ steps.benchmark-result.outputs.version }}" + destination="data/machine=$machine/version=$version" + mkdir -p "bench-history/$destination" "bench-history/latest/$machine" + cp "$result_path" "bench-history/$destination/benchmark.json" + cp "bench-history/$destination/benchmark.json" "bench-history/latest/$machine/benchmark.json" git -C bench-history add data latest git -C bench-history config user.name github-actions[bot] git -C bench-history config user.email 41898282+github-actions[bot]@users.noreply.github.com - git -C bench-history commit -m "Benchmark $BENCHMARK_VERSION" + git -C bench-history commit -m "Benchmark $version" git -C bench-history push origin HEAD:benchmark-results diff --git a/AGENTS.md b/AGENTS.md index 528c1b5a..4a8672bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ Machine partitions are derived from the execution environment, OS, CPU model, on The queries below use `clickhouse local`. If the ClickHouse binary is not installed, install it with `curl https://clickhouse.com/cli | sh` followed by `~/.local/bin/clickhousectl local use latest`. -Run benchmarks locally with `mix benchmark`. To include Benchee's built-in TProf pass, use `BENCHMARK_PROFILE=tprof mix benchmark`. JSON results are written below `bench/output/`, partitioned by detected machine specifications and SHA. Each file includes schema/run identity; timestamp and CI provenance; OS, architecture, CPU, cores, memory, Elixir, Erlang, and JIT details; benchmark configuration and units; and raw samples plus runtime, memory, and reduction statistics for every scenario. Override `BENCHMARK_MACHINE_PREFIX`, `BENCHMARK_VERSION`, `BENCHMARK_TIME`, or `BENCHMARK_WARMUP` when a reproducible series needs fixed settings; hardware fields are always detected. +Run benchmarks locally with `mix benchmark`. Use `mix benchmark --profile` to include Benchee's built-in TProf pass; CI profiles automatically. JSON results are written below `bench/output/`, partitioned by the detected machine specifications and current Git SHA. Each file includes schema/run identity; timestamp and CI provenance; OS, architecture, CPU, cores, memory, Elixir, Erlang, and JIT details; benchmark configuration and units; and raw samples plus runtime, memory, and reduction statistics for every scenario. An abridged result from the demo RowBinary benchmark looks like this. Actual files contain all samples and statistics, plus similarly shaped `memory_usage` and `reductions` collections for each scenario. diff --git a/bench/benchmark.exs b/bench/benchmark.exs index 0f5bf39a..186bf60c 100644 --- a/bench/benchmark.exs +++ b/bench/benchmark.exs @@ -29,28 +29,13 @@ inputs = end version = - System.get_env("BENCHMARK_VERSION") || - case System.cmd("git", ["rev-parse", "HEAD"], stderr_to_stdout: true) do - {sha, 0} -> String.trim(sha) - _ -> "working-tree" - end - -output_root = System.get_env("BENCHMARK_OUTPUT_DIR", "bench/output") - -seconds = fn name, default -> - case System.get_env(name) do - nil -> default - value -> String.to_float(value) + case System.cmd("git", ["rev-parse", "HEAD"], stderr_to_stdout: true) do + {sha, 0} -> String.trim(sha) + _ -> "working-tree" end -end -profile_after = - case System.get_env("BENCHMARK_PROFILE") do - nil -> false - "" -> false - "false" -> false - profiler -> String.to_existing_atom(profiler) - end +github_actions? = System.get_env("GITHUB_ACTIONS") == "true" +profile_after = if github_actions? or "--profile" in System.argv(), do: :tprof, else: false Benchee.run( %{ @@ -60,8 +45,8 @@ Benchee.run( end }, inputs: inputs, - time: seconds.("BENCHMARK_TIME", 5.0), - warmup: seconds.("BENCHMARK_WARMUP", 2.0), + time: 3, + warmup: 1, max_sample_size: 20_000, measure_function_call_overhead: true, pre_check: true, @@ -69,12 +54,8 @@ Benchee.run( formatters: [ {Benchee.Formatters.Console, extended_statistics: true}, {Ch.Bench.JSONFormatter, - output_root: output_root, + output_root: "bench/output", benchmark_version: version, - machine_prefix: - System.get_env( - "BENCHMARK_MACHINE_PREFIX", - if(System.get_env("CI") == "true", do: "ci", else: "local") - )} + machine_prefix: if(github_actions?, do: "github-actions", else: "local")} ] )