Add parser benchmark suite

.github/workflows/bump-version.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Compute next patch version
         id: next

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
     name: Format check
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
@@ -52,7 +52,7 @@ jobs:
           - {os: windows-latest,   compiler: msvc,        cc: "",       cxx: ""}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install Ninja (Linux)
         if: runner.os == 'Linux'

.github/workflows/deploy-nightly-docs.yml

@@ -22,7 +22,7 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 

.github/workflows/docs-preview.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - uses: cachix/install-nix-action@v27
         with:

.github/workflows/release-docs.yml

@@ -17,7 +17,7 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install clang
         run: sudo apt-get install -y clang

.github/workflows/windows-unicode.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - uses: lukka/get-cmake@latest
 

.gitignore

@@ -6,6 +6,7 @@
 
 # build artifacts
 build/
+build-bench/
 *.o
 *.a
 *.so

bench/CMakeLists.txt

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.20)
+
+project(cli20_bench LANGUAGES CXX)
+
+set(BENCH_CXX_STANDARD "20" CACHE STRING "C++ standard used for benchmark builds")
+set_property(CACHE BENCH_CXX_STANDARD PROPERTY STRINGS 20 26)
+
+set(CMAKE_CXX_STANDARD "${BENCH_CXX_STANDARD}")
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+include(cmake/BenchLibraries.cmake)
+
+add_subdirectory(runtime)
+add_subdirectory(size)

bench/cmake/BenchLibraries.cmake

@@ -0,0 +1,46 @@
+include_guard(GLOBAL)
+
+get_filename_component(CLI20_REPOSITORY_ROOT "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE)
+
+add_library(bench_common INTERFACE)
+target_include_directories(bench_common INTERFACE
+  "${CLI20_REPOSITORY_ROOT}/include"
+  "${CLI20_REPOSITORY_ROOT}/bench/include"
+)
+target_compile_features(bench_common INTERFACE cxx_std_20)
+
+add_library(bench_warnings INTERFACE)
+target_compile_options(bench_warnings INTERFACE
+  $<IF:$<CXX_COMPILER_ID:MSVC>,/W4,-Wall;-Wextra;-Wpedantic>
+)
+
+find_package(benchmark REQUIRED)
+if(POLICY CMP0167)
+  cmake_policy(SET CMP0167 NEW)
+endif()
+find_package(Boost REQUIRED COMPONENTS program_options)
+
+find_path(BENCH_CLI11_INCLUDE_DIR CLI/CLI.hpp REQUIRED)
+find_path(BENCH_ARGPARSE_INCLUDE_DIR argparse/argparse.hpp REQUIRED)
+find_path(BENCH_CXXOPTS_INCLUDE_DIR cxxopts.hpp REQUIRED)
+
+add_library(bench_cli11 INTERFACE)
+target_include_directories(bench_cli11 INTERFACE "${BENCH_CLI11_INCLUDE_DIR}")
+
+add_library(bench_argparse INTERFACE)
+target_include_directories(bench_argparse INTERFACE "${BENCH_ARGPARSE_INCLUDE_DIR}")
+
+add_library(bench_cxxopts INTERFACE)
+target_include_directories(bench_cxxopts INTERFACE "${BENCH_CXXOPTS_INCLUDE_DIR}")
+
+add_library(bench_boost_program_options INTERFACE)
+target_link_libraries(bench_boost_program_options INTERFACE Boost::program_options)
+
+function(bench_add_size_executable target source library_target)
+  add_executable("${target}" "${source}")
+  target_link_libraries("${target}" PRIVATE
+    bench_common
+    bench_warnings
+    "${library_target}"
+  )
+endfunction()

bench/compile_time/cases/many_options/argparse.cc

@@ -0,0 +1 @@
+#include "../../../size/many_options/argparse.cc"

bench/compile_time/cases/many_options/boost.cc

@@ -0,0 +1 @@
+#include "../../../size/many_options/boost.cc"

bench/compile_time/cases/many_options/cli11.cc

@@ -0,0 +1 @@
+#include "../../../size/many_options/cli11.cc"

bench/compile_time/cases/many_options/cli20.cc

@@ -0,0 +1 @@
+#include "../../../size/many_options/cli20.cc"

bench/compile_time/cases/many_options/cxxopts.cc

@@ -0,0 +1 @@
+#include "../../../size/many_options/cxxopts.cc"

bench/compile_time/cases/simple/argparse.cc

@@ -0,0 +1 @@
+#include "../../../size/simple/argparse.cc"

bench/compile_time/cases/simple/boost.cc

@@ -0,0 +1 @@
+#include "../../../size/simple/boost.cc"

bench/compile_time/cases/simple/cli11.cc

@@ -0,0 +1 @@
+#include "../../../size/simple/cli11.cc"

bench/compile_time/cases/simple/cli20.cc

@@ -0,0 +1 @@
+#include "../../../size/simple/cli20.cc"

bench/compile_time/cases/simple/cxxopts.cc

@@ -0,0 +1 @@
+#include "../../../size/simple/cxxopts.cc"

bench/compile_time/cases/subcommand/argparse.cc

@@ -0,0 +1 @@
+#include "../../../size/subcommand/argparse.cc"

bench/compile_time/cases/subcommand/boost.cc

@@ -0,0 +1 @@
+#include "../../../size/subcommand/boost.cc"

bench/compile_time/cases/subcommand/cli11.cc

@@ -0,0 +1 @@
+#include "../../../size/subcommand/cli11.cc"

bench/compile_time/cases/subcommand/cli20.cc

@@ -0,0 +1 @@
+#include "../../../size/subcommand/cli20.cc"

bench/compile_time/cases/subcommand/cxxopts.cc

@@ -0,0 +1 @@
+#include "../../../size/subcommand/cxxopts.cc"

bench/compile_time/scripts/compile-one.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$#" -ne 3 ]; then
+  echo "usage: $0 CASE LIBRARY OUTPUT_DIR" >&2
+  exit 2
+fi
+
+case_name="$1"
+library="$2"
+output_dir="$3"
+
+repo_root="$(cd "$(dirname "$0")/../../.." && pwd)"
+source_file="$repo_root/bench/compile_time/cases/$case_name/$library.cc"
+object_file="$output_dir/${case_name}_${library}.o"
+
+if [ ! -f "$source_file" ]; then
+  echo "unknown compile-time case: $case_name/$library" >&2
+  exit 2
+fi
+
+mkdir -p "$output_dir"
+
+: "${CXX:=c++}"
+: "${BENCH_CXX_STANDARD:=20}"
+: "${CXXFLAGS:=-std=c++$BENCH_CXX_STANDARD -O2}"
+
+exec "$CXX" $CXXFLAGS \
+  -I"$repo_root/include" \
+  -I"$repo_root/bench/include" \
+  -c "$source_file" \
+  -o "$object_file"

bench/compile_time/scripts/run.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "$0")/../../.." && pwd)"
+cxx_standard="${BENCH_CXX_STANDARD:-20}"
+out_dir="${1:-$repo_root/bench/results/compile_time/cxx$cxx_standard}"
+hyperfine_runs="${HYPERFINE_RUNS:-5}"
+hyperfine_warmup="${HYPERFINE_WARMUP:-1}"
+
+cases=(simple many_options subcommand)
+libraries=(cli20 cli11 argparse cxxopts boost)
+
+mkdir -p "$out_dir"
+result_file="$out_dir/results.tsv"
+json_dir="$out_dir/json"
+object_dir="$out_dir/objects"
+printf "standard\tcase\tlibrary\tseconds\n" > "$result_file"
+
+for case_name in "${cases[@]}"; do
+  for library in "${libraries[@]}"; do
+    mkdir -p "$json_dir" "$object_dir"
+    object_file="$object_dir/${case_name}_${library}.o"
+    json_file="$json_dir/${case_name}_${library}.json"
+    hyperfine \
+      --warmup "$hyperfine_warmup" \
+      --runs "$hyperfine_runs" \
+      --prepare "rm -f '$object_file'" \
+      --export-json "$json_file" \
+      "bash $repo_root/bench/compile_time/scripts/compile-one.sh '$case_name' '$library' '$object_dir'" \
+      >/dev/null
+    elapsed="$(python3 -c 'import json, sys; print("{:.6f}".format(json.load(open(sys.argv[1]))["results"][0]["mean"]))' "$json_file")"
+    printf "c++%s\t%s\t%s\t%s\n" "$cxx_standard" "$case_name" "$library" "$elapsed" |
+      tee -a "$result_file"
+  done
+done

bench/include/bench/allocation_counter.hh

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <benchmark/benchmark.h>
+
+#include <atomic>
+#include <cstddef>
+
+namespace bench {
+
+struct AllocationStats {
+  std::size_t allocations{};
+  std::size_t bytes{};
+};
+
+class AllocationCounter {
+ public:
+  AllocationCounter() {
+    reset();
+    enabled() = true;
+  }
+
+  AllocationCounter(const AllocationCounter&) = delete;
+  auto operator=(const AllocationCounter&) -> AllocationCounter& = delete;
+
+  ~AllocationCounter() { enabled() = false; }
+
+  static auto reset() -> void {
+    allocations().store(0, std::memory_order_relaxed);
+    bytes().store(0, std::memory_order_relaxed);
+  }
+
+  static auto record(std::size_t n) -> void {
+    if (!enabled()) return;
+    allocations().fetch_add(1, std::memory_order_relaxed);
+    bytes().fetch_add(n, std::memory_order_relaxed);
+  }
+
+  [[nodiscard]] static auto snapshot() -> AllocationStats {
+    return {
+        .allocations = allocations().load(std::memory_order_relaxed),
+        .bytes = bytes().load(std::memory_order_relaxed),
+    };
+  }
+
+ private:
+  static auto enabled() -> bool& {
+    static thread_local bool value = false;
+    return value;
+  }
+
+  static auto allocations() -> std::atomic<std::size_t>& {
+    static std::atomic<std::size_t> value{};
+    return value;
+  }
+
+  static auto bytes() -> std::atomic<std::size_t>& {
+    static std::atomic<std::size_t> value{};
+    return value;
+  }
+};
+
+inline auto set_allocation_counters(benchmark::State& state,
+                                    std::size_t allocations, std::size_t bytes)
+    -> void {
+  state.counters["allocs"] = benchmark::Counter(
+      static_cast<double>(allocations), benchmark::Counter::kAvgIterations);
+  state.counters["bytes"] = benchmark::Counter(
+      static_cast<double>(bytes), benchmark::Counter::kAvgIterations);
+}
+
+template <class Fn>
+auto measure_allocations(Fn&& fn) -> AllocationStats {
+  AllocationCounter counter;
+  fn();
+  return counter.snapshot();
+}
+
+}  // namespace bench

bench/include/bench/argv.hh

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace bench {
+
+struct Argv {
+  std::vector<std::string> storage;
+  std::vector<char*> raw;
+  std::vector<std::string_view> views;
+
+  Argv(std::initializer_list<std::string_view> args) {
+    storage.reserve(args.size());
+    raw.reserve(args.size());
+    views.reserve(args.size());
+    for (std::string_view arg : args) {
+      storage.emplace_back(arg);
+    }
+    refresh();
+  }
+
+  auto refresh() -> void {
+    raw.clear();
+    views.clear();
+    for (auto& arg : storage) {
+      raw.push_back(arg.data());
+      views.emplace_back(arg);
+    }
+  }
+
+  [[nodiscard]] auto argc() const -> int { return static_cast<int>(raw.size()); }
+
+  [[nodiscard]] auto argv() -> char** { return raw.data(); }
+
+  [[nodiscard]] auto span() const -> std::span<const std::string_view> {
+    return views;
+  }
+};
+
+}  // namespace bench