diff --git a/.formatter.exs b/.formatter.exs index 17f7637c..dc006466 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,5 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{lib,test}/**/*.{ex,exs}"], + inputs: ["{mix,.formatter}.exs", "{lib,test,bench}/**/*.{ex,exs}"], import_deps: [:stream_data] ] diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..17f5681d --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,133 @@ +name: benchmark + +on: + pull_request: + push: + branches: [master] + schedule: + - cron: "17 10 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: benchmark-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + row-binary: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - id: beam + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.19" + otp-version: "28" + + - uses: actions/cache@v6 + with: + path: | + deps + _build + key: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('mix.lock') }} + restore-keys: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}- + + - run: mix deps.get --only dev + - run: mix compile --warnings-as-errors + - run: mix benchmark + + - name: Locate benchmark result + id: benchmark-result + run: | + result_path=$(cat bench/output/current-result-path) + machine=$(jq --raw-output .machine "$result_path") + version=$(jq --raw-output .benchmark_version "$result_path") + echo "path=$result_path" >> "$GITHUB_OUTPUT" + echo "machine=$machine" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Install ClickHouse Local + env: + DO_NOT_TRACK: "1" + run: | + curl --fail --silent --show-error https://clickhouse.com/cli | sh + "$HOME/.local/bin/clickhousectl" local use latest + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Load benchmark history + id: benchmark-history + run: | + result_path="${{ steps.benchmark-result.outputs.path }}" + machine="${{ steps.benchmark-result.outputs.machine }}" + + if git fetch origin benchmark-results; then + git worktree add bench-history FETCH_HEAD + else + git worktree add --detach bench-history + git -C bench-history switch --orphan benchmark-results + fi + + mkdir -p bench/compare + cp "$result_path" bench/compare/current.json + + if test -f "bench-history/latest/$machine/benchmark.json"; then + cp "bench-history/latest/$machine/benchmark.json" bench/compare/baseline.json + else + cp bench/compare/current.json bench/compare/baseline.json + echo "note=No prior baseline exists; this run establishes it." >> "$GITHUB_OUTPUT" + fi + + - name: Compare with latest master benchmark + run: | + version="${{ steps.benchmark-result.outputs.version }}" + machine="${{ steps.benchmark-result.outputs.machine }}" + baseline_note="${{ steps.benchmark-history.outputs.note }}" + if test -z "$baseline_note"; then + baseline_note="Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise." + fi + clickhouse local --queries-file bench/compare.sql > bench/compare/comment.md + { + echo "## RowBinary benchmark" + echo + echo "$baseline_note" + echo + cat bench/compare/comment.md + echo + echo "Commit: \`$version\` · Machine: \`$machine\` · Profiler: TProf · Full JSON results are attached to this workflow run." + } > bench/compare/pr-comment.md + cat bench/compare/pr-comment.md + + - uses: actions/upload-artifact@v7 + with: + name: row-binary-benchmark-${{ steps.benchmark-result.outputs.version }} + path: ${{ steps.benchmark-result.outputs.path }} + + - name: Comment on pull request + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: gh pr comment "${{ github.event.pull_request.number }}" --edit-last --create-if-none --body-file bench/compare/pr-comment.md + + - name: Store master result + if: github.event_name == 'push' || github.event_name == 'schedule' + run: | + result_path="${{ steps.benchmark-result.outputs.path }}" + machine="${{ steps.benchmark-result.outputs.machine }}" + version="${{ steps.benchmark-result.outputs.version }}" + destination="data/machine=$machine/version=$version" + mkdir -p "bench-history/$destination" "bench-history/latest/$machine" + cp "$result_path" "bench-history/$destination/benchmark.json" + cp "bench-history/$destination/benchmark.json" "bench-history/latest/$machine/benchmark.json" + git -C bench-history add data latest + git -C bench-history config user.name github-actions[bot] + git -C bench-history config user.email 41898282+github-actions[bot]@users.noreply.github.com + git -C bench-history commit -m "Benchmark $version" + git -C bench-history push origin HEAD:benchmark-results diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..4a8672bd --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,83 @@ +# Benchmark history + +Performance results live on the orphan `benchmark-results` branch. They are stored as self-describing Benchee JSON under: + +```text +data/machine=/version=/benchmark.json +latest//benchmark.json +``` + +Machine partitions are derived from the execution environment, OS, CPU model, online core count, total memory, and architecture. CI partitions start with `github-actions-`, while local partitions start with `local-`. Never compare results across machine partitions. A benchmark version is the commit SHA whose code was measured. + +The queries below use `clickhouse local`. If the ClickHouse binary is not installed, install it with `curl https://clickhouse.com/cli | sh` followed by `~/.local/bin/clickhousectl local use latest`. + +Run benchmarks locally with `mix benchmark`. Use `mix benchmark --profile` to include Benchee's built-in TProf pass; CI profiles automatically. JSON results are written below `bench/output/`, partitioned by the detected machine specifications and current Git SHA. Each file includes schema/run identity; timestamp and CI provenance; OS, architecture, CPU, cores, memory, Elixir, Erlang, and JIT details; benchmark configuration and units; and raw samples plus runtime, memory, and reduction statistics for every scenario. + +An abridged result from the demo RowBinary benchmark looks like this. Actual files contain all samples and statistics, plus similarly shaped `memory_usage` and `reductions` collections for each scenario. + +```json +{ + "schema_version": 1, + "benchmark_version": "e19815a...", + "machine": "local-macOS-Apple-M2-8-cores-8-GB-aarch64-apple-darwin", + "generated_at": "2026-08-03T12:43:41Z", + "ci": false, + "units": { + "configuration_time": "nanosecond", + "run_time_samples": "nanosecond", + "run_time_ips": "iterations_per_second", + "memory_usage_samples": "byte", + "reductions_samples": "count" + }, + "system": { + "os": "macOS", + "architecture": "aarch64-apple-darwin", + "cpu_speed": "Apple M2", + "num_cores": 8, + "available_memory": "8 GB", + "elixir": "1.20.2", + "erlang": "29.0.3", + "jit_enabled?": true + }, + "configuration": { + "time": 20000000.0, + "warmup": 10000000.0, + "parallel": 1, + "percentiles": [50, 99], + "max_sample_size": 20000 + }, + "scenarios": [ + { + "job_name": "RowBinary.encode_rows/2", + "input_name": "1000 rows", + "run_time": { + "samples": [357000, 339500, 551708], + "statistics": { + "ips": 2485.05, + "average": 402405.88, + "median": 388562.5, + "std_dev_ratio": 0.1445, + "percentiles": {"25": 349646.0, "50": 388562.5, "75": 454552.5, "99": 555458.0}, + "sample_size": 50 + } + } + } + ] +} +``` + +To query history without modifying the working tree: + +```sh +git fetch origin benchmark-results +git worktree add .context/benchmark-results origin/benchmark-results +clickhouse local -q "SELECT _file, scenario.job_name, scenario.input_name, round(scenario.run_time.statistics.ips, 2) AS ips FROM file('.context/benchmark-results/data/machine=github-actions-*/version=*/benchmark.json', JSONEachRow) ARRAY JOIN scenarios AS scenario ORDER BY _file DESC, scenario.job_name, scenario.input_name FORMAT PrettyCompact" +``` + +For the same comparison used by CI, copy two result files to `bench/compare/baseline.json` and `bench/compare/current.json`, then run: + +```sh +clickhouse local --queries-file bench/compare.sql +``` + +The SQL treats an absolute throughput change below 5% as noise (`no material change`). Inspect raw samples in `scenario.run_time.samples` before attributing small changes to code. Remove the `.context/benchmark-results` worktree with `git worktree remove .context/benchmark-results` when finished. diff --git a/bench/benchmark.exs b/bench/benchmark.exs new file mode 100644 index 00000000..186bf60c --- /dev/null +++ b/bench/benchmark.exs @@ -0,0 +1,61 @@ +alias Ch.RowBinary + +Code.require_file("json_formatter.exs", __DIR__) + +types = ["UInt64", "String", "Array(UInt8)", "DateTime64(3, 'UTC')", "DateTime"] +titles = ["Golang SQL database driver", "Phoenix app event", "billing webhook payload"] +bytes = Enum.to_list(1..16) +base_datetime = DateTime.from_naive!(~N[2026-01-01 00:00:00.000], "Etc/UTC") +base_naive = ~N[2026-01-01 00:00:00] + +make_rows = fn count -> + Enum.map(1..count, fn i -> + [ + i, + Enum.at(titles, rem(i, length(titles))), + Enum.take(bytes, rem(i, length(bytes)) + 1), + DateTime.add(base_datetime, i * 17, :millisecond), + NaiveDateTime.add(base_naive, i, :second) + ] + end) +end + +inputs = + for count <- [1_000, 10_000], into: %{} do + rows = make_rows.(count) + + {"#{count} rows", + %{rows: rows, encoded: rows |> RowBinary.encode_rows(types) |> IO.iodata_to_binary()}} + end + +version = + case System.cmd("git", ["rev-parse", "HEAD"], stderr_to_stdout: true) do + {sha, 0} -> String.trim(sha) + _ -> "working-tree" + end + +github_actions? = System.get_env("GITHUB_ACTIONS") == "true" +profile_after = if github_actions? or "--profile" in System.argv(), do: :tprof, else: false + +Benchee.run( + %{ + "RowBinary.encode_rows/2" => fn %{rows: rows} -> RowBinary.encode_rows(rows, types) end, + "RowBinary.decode_rows/2" => fn %{encoded: encoded} -> + RowBinary.decode_rows(encoded, types) + end + }, + inputs: inputs, + time: 3, + warmup: 1, + max_sample_size: 20_000, + measure_function_call_overhead: true, + pre_check: true, + profile_after: profile_after, + formatters: [ + {Benchee.Formatters.Console, extended_statistics: true}, + {Ch.Bench.JSONFormatter, + output_root: "bench/output", + benchmark_version: version, + machine_prefix: if(github_actions?, do: "github-actions", else: "local")} + ] +) diff --git a/bench/compare.sql b/bench/compare.sql new file mode 100644 index 00000000..f0d75cf1 --- /dev/null +++ b/bench/compare.sql @@ -0,0 +1,44 @@ +SET join_use_nulls = 1; + +WITH +baseline AS ( + SELECT + scenario.job_name AS job_name, + scenario.input_name AS input_name, + scenario.run_time.statistics.ips AS baseline_ips, + scenario.run_time.statistics.median AS baseline_median_ns + FROM file('bench/compare/baseline.json', JSONEachRow) + ARRAY JOIN scenarios AS scenario +), +current AS ( + SELECT + scenario.job_name AS job_name, + scenario.input_name AS input_name, + scenario.run_time.statistics.ips AS current_ips, + scenario.run_time.statistics.median AS current_median_ns + FROM file('bench/compare/current.json', JSONEachRow) + ARRAY JOIN scenarios AS scenario +) +SELECT + coalesce(current.job_name, baseline.job_name) AS benchmark, + coalesce(current.input_name, baseline.input_name) AS input, + round(baseline_ips, 2) AS baseline_ips, + round(current_ips, 2) AS current_ips, + multiIf( + baseline_ips IS NULL, 'new', + current_ips IS NULL, 'removed', + abs((current_ips / baseline_ips - 1) * 100) < 5, 'no material change', + current_ips > baseline_ips, 'faster', + 'slower' + ) AS result, + if( + baseline_ips IS NULL OR current_ips IS NULL, + NULL, + round((current_ips / baseline_ips - 1) * 100, 2) + ) AS change_percent, + round(baseline_median_ns / 1000000, 3) AS baseline_median_ms, + round(current_median_ns / 1000000, 3) AS current_median_ms +FROM baseline +FULL OUTER JOIN current USING (job_name, input_name) +ORDER BY benchmark, input +FORMAT Markdown; diff --git a/bench/json_formatter.exs b/bench/json_formatter.exs new file mode 100644 index 00000000..e3b443fc --- /dev/null +++ b/bench/json_formatter.exs @@ -0,0 +1,100 @@ +defmodule Ch.Bench.JSONFormatter do + @behaviour Benchee.Formatter + + @configuration_fields [ + :time, + :warmup, + :memory_time, + :reduction_time, + :parallel, + :percentiles, + :max_sample_size, + :exclude_outliers, + :measure_function_call_overhead + ] + + @impl Benchee.Formatter + def format(%Benchee.Suite{} = suite, opts) do + version = Map.fetch!(opts, :benchmark_version) + + system = + suite.system + |> Map.from_struct() + |> Map.put(:architecture, :erlang.system_info(:system_architecture) |> List.to_string()) + + machine = machine_key(Map.fetch!(opts, :machine_prefix), system) + output_root = Map.fetch!(opts, :output_root) + + path = + Path.join([output_root, "machine=#{machine}", "version=#{safe(version)}", "benchmark.json"]) + + data = + %{ + schema_version: 1, + benchmark_version: version, + machine: machine, + generated_at: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601(), + ci: System.get_env("CI") == "true", + units: %{ + configuration_time: "nanosecond", + run_time_samples: "nanosecond", + run_time_ips: "iterations_per_second", + memory_usage_samples: "byte", + reductions_samples: "count" + }, + system: system, + configuration: Map.take(Map.from_struct(suite.configuration), @configuration_fields), + scenarios: Enum.map(suite.scenarios, &scenario/1) + } + |> json_safe() + |> Jason.encode_to_iodata!() + + %{data: data, path: path, output_root: output_root} + end + + @impl Benchee.Formatter + def write(%{data: data, path: path, output_root: output_root}, _opts) do + path |> Path.dirname() |> File.mkdir_p!() + File.write!(path, data) + File.write!(Path.join(output_root, "current-result-path"), path) + IO.puts("Benchmark artifact: #{path}") + end + + defp machine_key(prefix, system) do + [ + prefix, + system.os, + system.cpu_speed, + "#{system.num_cores}-cores", + system.available_memory, + system.architecture + ] + |> Enum.join("-") + |> safe() + end + + defp safe(value), do: String.replace(value, ~r/[^a-zA-Z0-9._-]/, "-") + + defp scenario(scenario) do + %{ + name: scenario.name, + job_name: scenario.job_name, + input_name: scenario.input_name, + tag: scenario.tag, + run_time: scenario.run_time_data, + memory_usage: scenario.memory_usage_data, + reductions: scenario.reductions_data + } + end + + defp json_safe(%_{} = struct), do: struct |> Map.from_struct() |> json_safe() + + defp json_safe(map) when is_map(map) do + Map.new(map, fn {key, value} -> {to_string(key), json_safe(value)} end) + end + + defp json_safe(list) when is_list(list), do: Enum.map(list, &json_safe/1) + defp json_safe(tuple) when is_tuple(tuple), do: tuple |> Tuple.to_list() |> json_safe() + defp json_safe(atom) when is_atom(atom) and atom not in [true, false, nil], do: to_string(atom) + defp json_safe(value), do: value +end diff --git a/mix.exs b/mix.exs index d7fcc923..eab4a8b5 100644 --- a/mix.exs +++ b/mix.exs @@ -11,6 +11,7 @@ defmodule Ch.MixProject do elixir: "~> 1.15", elixirc_paths: elixirc_paths(Mix.env()), deps: deps(), + aliases: aliases(), name: "Ch", description: "HTTP ClickHouse driver for Elixir", docs: docs(), @@ -25,6 +26,10 @@ defmodule Ch.MixProject do [extra_applications: [:logger | extra_applications(Mix.env())]] end + def cli do + [preferred_envs: [benchmark: :dev]] + end + # Specifies which paths to compile per environment. defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_env), do: ["lib"] @@ -41,6 +46,7 @@ defmodule Ch.MixProject do {:jason, "~> 1.0"}, {:decimal, "~> 3.0"}, {:ecto, "~> 3.14.0", optional: true}, + {:benchee, "~> 1.5", only: :dev}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :docs}, {:tz, "~> 0.28.1", only: [:dev, :test]}, @@ -48,6 +54,10 @@ defmodule Ch.MixProject do ] end + defp aliases do + [benchmark: "run bench/benchmark.exs"] + end + defp docs do [ source_url: @source_url, diff --git a/mix.lock b/mix.lock index 27e1b2c3..75cffab6 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,8 @@ %{ + "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"}, "db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"}, "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, + "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "ecto": {:hex, :ecto, "3.14.1", "7b740d87bdf45996aa0c2c2e081640906f10caa7ce5ba328fd294c7d49d0cc6f", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "24b991956796700f467d0a3ef3d303138a3ef9ddddf8b98f43758ee067b20a30"}, @@ -13,6 +15,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, "mint": {:hex, :mint, "1.9.3", "3337184d69179695c7a9f1714d92c11e629d36c8c037a21cf490131d3d150554", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5f7c9342480c069dbbc4eeac3490303c9e01870ff01a7f1d29b6107054fc1e74"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "tz": {:hex, :tz, "0.28.2", "6c47f3d1a8ee5c33a1d8f0ba49e5a851b0a30c408a587d907aff6e71228b3b32", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:mint, "~> 1.6", [hex: :mint, repo: "hexpm", optional: true]}], "hexpm", "a6bf7355a33f0a7511602ab4566432ac0901d8abff81e94e455bca19708a0c87"},