From 8fda508b141d2d9037213be10c090d9e9813da97 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 10:49:10 +0000 Subject: [PATCH 1/2] Add CI check that CodSpeed benchmarks stay under the 1ms iteration budget The benchmarking guide asks that each benchmark iteration complete in under 1ms, but nothing enforced it. CodSpeed runs in simulation mode, which estimates cycles from an instruction trace and never reports wall-clock time, so the existing job cannot answer the question. The `divan` dependency is really `codspeed-divan-compat`. Built without `--cfg codspeed` it re-exports CodSpeed's patched divan, which dumps per-iteration statistics as JSON when `CODSPEED_ENV` is set. The new `bench-budget` job rebuilds the sharded benchmarks in walltime mode, runs them once outside the CodSpeed runner, and compares the fastest observed iteration against the budget. Using the minimum rather than the median keeps a loaded shared runner from making the check flaky. Only benchmarks CodSpeed actually measures are checked. That set comes from replaying the analysis-mode binaries, which print one line per benchmark they run; because that is the same build CodSpeed uses, benchmarks gated with `#[cfg(not(codspeed))]` are excluded automatically rather than through an allowlist that would drift. Both modes generate benchmark URIs with identical code, so the two sets join exactly. Results are posted as a sticky PR comment rather than failing the build, and the job only runs on pull requests that touch a `benches/` directory. Signed-off-by: Joe Isaacs --- .github/workflows/ci.yml | 5 + .github/workflows/codspeed.yml | 125 +++++++++++ docs/developer-guide/benchmarking.md | 18 ++ scripts/check-bench-budget.py | 265 +++++++++++++++++++++++ scripts/tests/test_check_bench_budget.py | 210 ++++++++++++++++++ 5 files changed, 623 insertions(+) create mode 100644 scripts/check-bench-budget.py create mode 100644 scripts/tests/test_check_bench_budget.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86fa85fd4d6..d44816b5431 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,6 +166,11 @@ jobs: uv run --no-project --with pytest --with xxhash \ pytest scripts/tests/test_measurement_id.py + - name: Pytest - benchmark budget check + run: | + uv run --no-project --with pytest \ + pytest scripts/tests/test_check_bench_budget.py + python-cuda-test: name: "Python CUDA (test)" if: github.repository == 'vortex-data/vortex' diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d27a977e11a..ade09935262 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -33,6 +33,7 @@ jobs: pull-requests: read outputs: run-cuda-benchmarks: ${{ github.event_name != 'pull_request' || steps.filter.outputs.cuda == 'true' }} + run-budget-check: ${{ steps.filter.outputs.benches == 'true' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 @@ -44,8 +45,13 @@ jobs: - "vortex-cuda/**" # Only this workflow defines the CUDA benchmark jobs. - ".github/workflows/codspeed.yml" + benches: + - "**/benches/**" + - "scripts/check-bench-budget.py" + - ".github/workflows/codspeed.yml" bench-codspeed: + needs: [changes] strategy: matrix: include: @@ -91,6 +97,125 @@ jobs: token: ${{ secrets.CODSPEED_TOKEN }} mode: "simulation" + # The analysis-mode binaries print one `Measured:`/`Checked:` line per benchmark they + # run, which is the only authoritative answer to "what does CodSpeed actually + # benchmark?" -- it honours `#[cfg(not(codspeed))]` because it *is* that build. Replay + # them outside the CodSpeed runner (one un-instrumented iteration each, so this costs + # seconds) to hand the budget job its scope. + - name: Enumerate measured benchmarks + if: needs.changes.outputs.run-budget-check == 'true' + run: | + set -Eeuo pipefail + cargo codspeed run > codspeed-run.log + grep -E "^(Measured|Checked): " codspeed-run.log > codspeed-uris.txt + wc -l < codspeed-uris.txt + + - name: Upload measured benchmark URIs + if: needs.changes.outputs.run-budget-check == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: codspeed-uris-${{ matrix.shard }} + path: codspeed-uris.txt + retention-days: 1 + if-no-files-found: error + + # Enforces the "keep per-iteration execution time under ~1 ms" rule from + # docs/developer-guide/benchmarking.md. The simulation instrument above reports estimated + # cycles from an instruction trace, never wall-clock time, so the budget needs its own + # measurement: the same benchmarks rebuilt in walltime mode, where the divan harness dumps + # per-iteration statistics as JSON. Results are reported as a PR comment, not a hard + # failure. + bench-budget: + needs: [changes, bench-codspeed] + if: github.event_name == 'pull_request' && needs.changes.outputs.run-budget-check == 'true' + strategy: + matrix: + include: + - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error vortex-mask vortex-compute" } + - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" } + - { shard: 3, name: "Main library", packages: "vortex" } + - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" } + - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" } + - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" } + - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } + - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks vortex-row" } + name: "Check benchmark budget (Shard #${{ matrix.shard }})" + timeout-minutes: 30 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=bench-budget-{1}', github.run_id, matrix.shard) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + - name: Install Codspeed + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 + with: + tool: cargo-codspeed + - name: Download measured benchmark URIs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: codspeed-uris-${{ matrix.shard }} + - name: Build benchmarks + env: + RUSTFLAGS: "-C target-feature=+avx2" + run: cargo codspeed build -m walltime ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench + # `cargo codspeed run` only asks the harness to dump raw walltime JSON when it believes + # it is under the CodSpeed runner, so set CODSPEED_ENV ourselves. No token, no upload, + # and no valgrind: these numbers are for the budget check alone and are never reported + # to CodSpeed. Benchmarks that honour the budget make this run take milliseconds. + - name: Measure benchmarks + env: + CODSPEED_ENV: "1" + DIVAN_SAMPLE_COUNT: "3" + DIVAN_MIN_TIME: "0" + run: cargo codspeed run -m walltime + - name: Check against budget + run: | + python3 scripts/check-bench-budget.py check \ + --scope codspeed-uris.txt \ + --shard "${{ matrix.shard }}" \ + --output "verdicts/${{ matrix.shard }}.json" + - name: Upload verdict + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bench-budget-${{ matrix.shard }} + path: verdicts/ + retention-days: 1 + if-no-files-found: error + + bench-budget-comment: + needs: [bench-budget] + if: always() && needs.bench-budget.result == 'success' && github.event.pull_request.head.repo.fork == false + name: "Report benchmark budget" + timeout-minutes: 10 + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Download verdicts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: bench-budget-* + path: verdicts + - name: Render comment + run: | + python3 scripts/check-bench-budget.py report --inputs verdicts --output comment.md + cat comment.md >> "$GITHUB_STEP_SUMMARY" + - name: Comment PR + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + with: + file-path: comment.md + comment-tag: bench-budget-comment + # Getting a GPU box is slow, in the future we can build on a box without one and only run # on GPU machines. bench-codspeed-cuda-build: diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index 1655d7f8a08..86c77e680a7 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -124,6 +124,24 @@ fn my_bench(bencher: Bencher, num_indices: usize) { ... } Each individual iteration of the benchmarked closure should complete in **less than 1ms**. This is to keep benchmarks snappy, locally and on CI. +CI checks this on any pull request that touches a `benches/` directory, and reports the +result as a comment rather than failing the build. Because the simulation instrument +reports estimated cycles rather than wall-clock time, the `bench-budget` job in +`.github/workflows/codspeed.yml` rebuilds the same benchmarks in CodSpeed's walltime mode +and compares the *fastest* observed iteration against the budget -- the estimate least +affected by runner noise. Only benchmarks CodSpeed actually measures are checked, so +anything gated with `#[cfg(not(codspeed))]` is exempt automatically. + +To reproduce the check locally: + +```bash +cargo codspeed build -m walltime -p --profile bench +CODSPEED_ENV=1 cargo codspeed run -m walltime +python3 scripts/check-bench-budget.py check --shard local --output verdicts/local.json +``` + +Omitting `--scope` checks every benchmark, including the `#[cfg(not(codspeed))]` ones. + ### Gate CodSpeed-incompatible benchmarks Use `#[cfg(not(codspeed))]` for benchmarks that are incompatible with CodSpeed. diff --git a/scripts/check-bench-budget.py b/scripts/check-bench-budget.py new file mode 100644 index 00000000000..e3dbe212665 --- /dev/null +++ b/scripts/check-bench-budget.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Check that CodSpeed benchmarks stay under a per-iteration wall-clock budget. + +CodSpeed itself runs in ``simulation`` mode, which executes each benchmark exactly once +and estimates cycles from an instruction trace -- it never reports wall-clock time, so it +cannot enforce the "keep per-iteration execution time under ~1 ms" rule from +``docs/developer-guide/benchmarking.md``. + +The ``divan`` dependency is really ``codspeed-divan-compat``. Built *without* ``--cfg +codspeed`` it re-exports CodSpeed's patched divan, which writes one JSON file per +benchmark to ``target/codspeed/walltime/raw_results/divan/`` whenever ``CODSPEED_ENV`` is +set. Those files carry per-iteration statistics (the harness already divides each round by +its iteration count), which is exactly the quantity the budget is written against. + +Two subcommands: + +``check`` + Read the raw walltime results for one shard, restrict them to the benchmarks CodSpeed + actually measures, and emit a JSON verdict. + +``report`` + Merge the per-shard verdicts into a single Markdown comment body. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +DEFAULT_MAX_NS = 1_000_000 +"""One millisecond, per the benchmarking guide.""" + +GUIDE_URL = ( + "https://github.com/vortex-data/vortex/blob/develop/docs/developer-guide/benchmarking.md" + "#keep-per-iteration-execution-time-under-1-ms" +) + +MAX_ROWS = 30 +"""Cap the table so a wholesale regression cannot produce an unreadable comment.""" + + +def format_duration(nanos: float) -> str: + """Render a nanosecond count using the same units as divan's own output.""" + for limit, unit, scale in ( + (1_000, "ns", 1), + (1_000_000, "µs", 1_000), + (1_000_000_000, "ms", 1_000_000), + ): + if nanos < limit: + return f"{nanos / scale:.3g} {unit}" + return f"{nanos / 1_000_000_000:.3g} s" + + +def load_raw_results(raw_results: Path) -> list[dict]: + """Load every per-benchmark JSON file written by the patched divan harness.""" + benchmarks = [] + for path in sorted(raw_results.glob("**/*.json")): + with path.open() as f: + benchmarks.append(json.load(f)) + return benchmarks + + +def load_scope(scope: Path | None) -> set[str] | None: + """Load the set of benchmark URIs CodSpeed measures, or ``None`` for "all of them". + + The URIs are produced by the analysis-mode binaries, which print ``Measured: `` + (instrumented) or ``Checked: `` (not instrumented) for every benchmark they run. + Both modes build that URI with identical code, so the strings match the ``uri`` field + in the walltime results exactly. + """ + if scope is None: + return None + uris = set() + for line in scope.read_text().splitlines(): + line = line.strip() + for prefix in ("Measured: ", "Checked: "): + if line.startswith(prefix): + line = line[len(prefix) :] + break + # Instrumented runs append a group suffix that the URI itself does not carry. + line = line.split(" (group: ", 1)[0].strip() + if line: + uris.add(line) + return uris + + +def check(args: argparse.Namespace) -> int: + benchmarks = load_raw_results(args.raw_results) + scope = load_scope(args.scope) + + in_scope, violations = [], [] + for bench in benchmarks: + uri = bench["uri"] + if scope is not None and uri not in scope: + continue + in_scope.append(uri) + nanos = bench["stats"][args.metric] + if nanos > args.max_ns: + violations.append({"uri": uri, "name": bench["name"], "ns": nanos}) + + violations.sort(key=lambda v: v["ns"], reverse=True) + verdict = { + "shard": args.shard, + "max_ns": args.max_ns, + "metric": args.metric, + "measured": len(benchmarks), + "in_scope": len(in_scope), + "violations": violations, + } + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(verdict, indent=2) + "\n") + + print( + f"shard {args.shard}: {len(in_scope)}/{len(benchmarks)} benchmarks in scope, " + f"{len(violations)} over the {format_duration(args.max_ns)} budget" + ) + for violation in violations: + print(f" {format_duration(violation['ns']):>10} {violation['uri']}") + + return 1 if violations and args.fail_on_violation else 0 + + +def render_report(verdicts: list[dict]) -> str: + """Render the merged verdicts as the body of a sticky PR comment.""" + in_scope = sum(v["in_scope"] for v in verdicts) + violations = [v for verdict in verdicts for v in verdict["violations"]] + violations.sort(key=lambda v: v["ns"], reverse=True) + + # Every shard is configured identically; fall back to the default if none ran. + max_ns = verdicts[0]["max_ns"] if verdicts else DEFAULT_MAX_NS + metric = verdicts[0]["metric"] if verdicts else "min_ns" + budget = format_duration(max_ns) + + lines = ["## ⏱️ Benchmark iteration budget", ""] + + if not violations: + lines += [ + f"`✅ {in_scope}` CodSpeed benchmarks are within the **{budget}** " + "per-iteration budget.", + ] + else: + lines += [ + f"`⚠️ {len(violations)}` of `{in_scope}` CodSpeed benchmarks exceed the " + f"**{budget}** per-iteration budget.", + "", + "CodSpeed's simulation instrument runs each benchmark exactly once, so a slow " + "iteration costs CI time without buying any extra signal. Shrink the input " + f"size, or gate the benchmark with `#[cfg(not(codspeed))]`. See [the " + f"benchmarking guide]({GUIDE_URL}).", + "", + "| Benchmark | Fastest iteration | Over budget |", + "| --- | --- | --- |", + ] + for violation in violations[:MAX_ROWS]: + over = violation["ns"] / max_ns + lines.append( + f"| `{violation['uri']}` | {format_duration(violation['ns'])} | {over:.1f}× |" + ) + if len(violations) > MAX_ROWS: + lines += [ + "", + f"> ℹ️ _Only the first {MAX_ROWS} of {len(violations)} benchmarks are " + "displayed._", + ] + + lines += [ + "", + "
How this is measured", + "", + "Benchmarks are rebuilt in CodSpeed's walltime mode and run once outside the " + f"CodSpeed runner. The reported number is `{metric}` -- the fastest observed " + "iteration, which is the estimate least contaminated by runner noise, so a shared " + "CI machine cannot make this check flaky.", + "", + "Only benchmarks that CodSpeed actually measures are checked. That set comes from " + "the analysis-mode binaries built by `cargo codspeed build`, which enumerate every " + "benchmark they run, so anything behind `#[cfg(not(codspeed))]` is excluded " + "automatically rather than by an allowlist that can drift.", + "", + "
", + ] + return "\n".join(lines) + "\n" + + +def report(args: argparse.Namespace) -> int: + verdicts = [] + for path in sorted(args.inputs.glob("**/*.json")): + with path.open() as f: + verdicts.append(json.load(f)) + + if not verdicts: + print(f"no verdicts found under {args.inputs}", file=sys.stderr) + return 1 + + body = render_report(verdicts) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(body) + print(body) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + check_parser = subparsers.add_parser( + "check", help="check one shard's walltime results against the budget" + ) + check_parser.add_argument( + "--raw-results", + type=Path, + default=Path("target/codspeed/walltime/raw_results/divan"), + help="directory of per-benchmark JSON files written by the divan harness", + ) + check_parser.add_argument( + "--scope", + type=Path, + help="file listing the benchmark URIs CodSpeed measures; all are checked if omitted", + ) + check_parser.add_argument( + "--max-ns", + type=int, + default=DEFAULT_MAX_NS, + help=f"per-iteration budget in nanoseconds (default: {DEFAULT_MAX_NS})", + ) + check_parser.add_argument( + "--metric", + choices=["min_ns", "median_ns", "mean_ns", "max_ns"], + default="min_ns", + help="statistic to compare against the budget (default: min_ns)", + ) + check_parser.add_argument("--shard", default="", help="shard name, for reporting") + check_parser.add_argument( + "--output", type=Path, required=True, help="path to write the JSON verdict to" + ) + check_parser.add_argument( + "--fail-on-violation", + action="store_true", + help="exit non-zero when a benchmark is over budget (default: report only)", + ) + check_parser.set_defaults(func=check) + + report_parser = subparsers.add_parser( + "report", help="merge per-shard verdicts into a Markdown comment" + ) + report_parser.add_argument( + "--inputs", type=Path, required=True, help="directory of JSON verdicts" + ) + report_parser.add_argument( + "--output", type=Path, required=True, help="path to write the Markdown body to" + ) + report_parser.set_defaults(func=report) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_check_bench_budget.py b/scripts/tests/test_check_bench_budget.py new file mode 100644 index 00000000000..a38a2951012 --- /dev/null +++ b/scripts/tests/test_check_bench_budget.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +import importlib.util +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +BUDGET_SCRIPT = REPO_ROOT / "scripts" / "check-bench-budget.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("check_bench_budget", BUDGET_SCRIPT) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +budget = load_module() + + +def write_raw_result(directory: Path, uri: str, min_ns: float, name: str | None = None) -> None: + """Write a raw result in the shape the patched divan harness emits.""" + directory.mkdir(parents=True, exist_ok=True) + payload = { + "name": name or uri.rsplit("::", 1)[-1], + "uri": uri, + "config": {}, + "stats": { + "min_ns": min_ns, + "max_ns": min_ns * 1.2, + "mean_ns": min_ns * 1.1, + "median_ns": min_ns * 1.05, + "stdev_ns": 0.0, + "q1_ns": min_ns, + "q3_ns": min_ns * 1.1, + "rounds": 3, + "total_time": 0.1, + "iqr_outlier_rounds": 0, + "stdev_outlier_rounds": 0, + "iter_per_round": 1, + "warmup_iters": 0, + }, + } + (directory / f"{abs(hash(uri))}.json").write_text(json.dumps(payload)) + + +def run_check(tmp_path: Path, **overrides): + raw = tmp_path / "raw" + output = tmp_path / "verdict.json" + args = { + "raw_results": raw, + "scope": None, + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "shard": "1", + "output": output, + "fail_on_violation": False, + } + args.update(overrides) + code = budget.check(_Namespace(**args)) + return code, json.loads(output.read_text()) + + +class _Namespace: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +@pytest.mark.parametrize( + ("nanos", "expected"), + [(36, "36 ns"), (61_600, "61.6 µs"), (14_100_000, "14.1 ms"), (2_000_000_000, "2 s")], +) +def test_format_duration_matches_divan_units(nanos, expected): + assert budget.format_duration(nanos) == expected + + +def test_check_flags_only_over_budget_benchmarks(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "vortex-mask/benches/rank.rs::fast", 36) + write_raw_result(raw, "vortex-geo/benches/envelope.rs::slow", 65_500_000) + + code, verdict = run_check(tmp_path) + + assert code == 0 + assert verdict["in_scope"] == 2 + assert [v["uri"] for v in verdict["violations"]] == ["vortex-geo/benches/envelope.rs::slow"] + + +def test_check_ignores_benchmarks_codspeed_does_not_measure(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "vortex/benches/throughput.rs::gated", 65_500_000) + write_raw_result(raw, "vortex-mask/benches/rank.rs::fast", 36) + scope = tmp_path / "scope.txt" + scope.write_text("Measured: vortex-mask/benches/rank.rs::fast\n") + + code, verdict = run_check(tmp_path, scope=scope) + + assert code == 0 + assert verdict["measured"] == 2 + assert verdict["in_scope"] == 1 + assert verdict["violations"] == [] + + +def test_check_can_fail_the_job(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "vortex-geo/benches/envelope.rs::slow", 65_500_000) + + code, _ = run_check(tmp_path, fail_on_violation=True) + + assert code == 1 + + +def test_check_orders_violations_worst_first(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "a.rs::mid", 14_100_000) + write_raw_result(raw, "a.rs::worst", 2_000_000_000) + write_raw_result(raw, "a.rs::least", 1_000_001) + + _, verdict = run_check(tmp_path) + + assert [v["uri"] for v in verdict["violations"]] == ["a.rs::worst", "a.rs::mid", "a.rs::least"] + + +@pytest.mark.parametrize( + "line", + [ + "Measured: vortex-mask/benches/rank.rs::fast", + "Checked: vortex-mask/benches/rank.rs::fast", + "Measured: vortex-mask/benches/rank.rs::fast (group: outer/inner)", + " Measured: vortex-mask/benches/rank.rs::fast ", + ], +) +def test_load_scope_parses_harness_output(tmp_path, line): + scope = tmp_path / "scope.txt" + scope.write_text(f"{line}\n") + + assert budget.load_scope(scope) == {"vortex-mask/benches/rank.rs::fast"} + + +def test_load_scope_of_none_means_check_everything(): + assert budget.load_scope(None) is None + + +def test_report_merges_shards_and_sorts_globally(): + body = budget.render_report( + [ + { + "shard": "1", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 2, + "in_scope": 2, + "violations": [{"uri": "a.rs::mid", "name": "mid", "ns": 14_100_000}], + }, + { + "shard": "2", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 3, + "in_scope": 3, + "violations": [{"uri": "b.rs::worst", "name": "worst", "ns": 2_000_000_000}], + }, + ] + ) + + assert "`⚠️ 2` of `5` CodSpeed benchmarks exceed" in body + assert body.index("b.rs::worst") < body.index("a.rs::mid") + assert "| `b.rs::worst` | 2 s | 2000.0× |" in body + + +def test_report_is_reassuring_when_clean(): + body = budget.render_report( + [ + { + "shard": "1", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 7, + "in_scope": 7, + "violations": [], + } + ] + ) + + assert "`✅ 7` CodSpeed benchmarks are within the **1 ms** per-iteration budget." in body + assert "| Benchmark |" not in body + + +def test_report_truncates_a_wholesale_regression(): + violations = [{"uri": f"a.rs::b{i}", "name": f"b{i}", "ns": 2_000_000 + i} for i in range(50)] + body = budget.render_report( + [ + { + "shard": "1", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 50, + "in_scope": 50, + "violations": violations, + } + ] + ) + + assert body.count("| `a.rs::") == budget.MAX_ROWS + assert f"Only the first {budget.MAX_ROWS} of 50 benchmarks are displayed." in body From 40c39407a3b93257dcd9839263839ff35d2343e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 14:01:09 +0000 Subject: [PATCH 2/2] TEMPORARY: add an over-budget benchmark to exercise the budget check Adds a benchmark that deliberately breaks the 1ms per-iteration rule, so the bench-budget job has something to catch on this pull request. Counting set bits across a 32 MiB bit buffer is memory-bandwidth bound and measures 3.28ms locally, against 71 existing vortex-mask benchmarks that all pass. The work is real rather than a sleep: CodSpeed's simulation instrument excludes system calls, so a sleeping benchmark would appear free there while still consuming CI wall-clock time. This commit must be reverted before merge. Signed-off-by: Joe Isaacs --- vortex-mask/Cargo.toml | 5 ++++ vortex-mask/benches/budget_canary.rs | 38 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 vortex-mask/benches/budget_canary.rs diff --git a/vortex-mask/Cargo.toml b/vortex-mask/Cargo.toml index cf681101d43..2076f56586b 100644 --- a/vortex-mask/Cargo.toml +++ b/vortex-mask/Cargo.toml @@ -45,5 +45,10 @@ harness = false name = "mask_iteration" harness = false +# TEMPORARY: proves the bench-budget CI check fires. Delete before merge. +[[bench]] +name = "budget_canary" +harness = false + [lints] workspace = true diff --git a/vortex-mask/benches/budget_canary.rs b/vortex-mask/benches/budget_canary.rs new file mode 100644 index 00000000000..94713e0987e --- /dev/null +++ b/vortex-mask/benches/budget_canary.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! TEMPORARY: a deliberately over-budget benchmark, used to prove that the `bench-budget` +//! CI job actually detects and reports a violation. +//! +//! This exists only to exercise the check on the pull request that introduces it, and +//! **must be deleted before merge**. It intentionally violates the "keep per-iteration +//! execution time under ~1 ms" rule in `docs/developer-guide/benchmarking.md`. +//! +//! The work is real rather than a `sleep`, because CodSpeed's simulation instrument +//! excludes system calls -- a sleeping benchmark would look free there while still +//! burning CI wall-clock time. + +#![allow(clippy::unwrap_used, clippy::cast_possible_truncation)] + +use divan::Bencher; +use vortex_buffer::BitBuffer; +use vortex_mask::Mask; + +fn main() { + divan::main(); +} + +/// 256Mi bits is a 32 MiB bit buffer. Counting the set bits in a prefix of it is bound by +/// memory bandwidth, which puts a single iteration well past the 1 ms budget on any runner. +const CANARY_LEN: usize = 256 * 1024 * 1024; + +#[divan::bench] +fn over_budget_prefix_count(bencher: Bencher) { + let mask = Mask::from_buffer(BitBuffer::from_iter( + (0..CANARY_LEN).map(|i| (i * 7 + 13) % 1000 < 900), + )); + let indices = [CANARY_LEN / 4, CANARY_LEN - CANARY_LEN / 8]; + bencher + .with_inputs(|| (&mask, indices)) + .bench_refs(|(mask, indices)| mask.valid_counts_for_indices(indices)); +}