diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86fa85fd4d6..d44816b5431 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,6 +166,11 @@ jobs: uv run --no-project --with pytest --with xxhash \ pytest scripts/tests/test_measurement_id.py + - name: Pytest - benchmark budget check + run: | + uv run --no-project --with pytest \ + pytest scripts/tests/test_check_bench_budget.py + python-cuda-test: name: "Python CUDA (test)" if: github.repository == 'vortex-data/vortex' diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d27a977e11a..ade09935262 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -33,6 +33,7 @@ jobs: pull-requests: read outputs: run-cuda-benchmarks: ${{ github.event_name != 'pull_request' || steps.filter.outputs.cuda == 'true' }} + run-budget-check: ${{ steps.filter.outputs.benches == 'true' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 @@ -44,8 +45,13 @@ jobs: - "vortex-cuda/**" # Only this workflow defines the CUDA benchmark jobs. - ".github/workflows/codspeed.yml" + benches: + - "**/benches/**" + - "scripts/check-bench-budget.py" + - ".github/workflows/codspeed.yml" bench-codspeed: + needs: [changes] strategy: matrix: include: @@ -91,6 +97,125 @@ jobs: token: ${{ secrets.CODSPEED_TOKEN }} mode: "simulation" + # The analysis-mode binaries print one `Measured:`/`Checked:` line per benchmark they + # run, which is the only authoritative answer to "what does CodSpeed actually + # benchmark?" -- it honours `#[cfg(not(codspeed))]` because it *is* that build. Replay + # them outside the CodSpeed runner (one un-instrumented iteration each, so this costs + # seconds) to hand the budget job its scope. + - name: Enumerate measured benchmarks + if: needs.changes.outputs.run-budget-check == 'true' + run: | + set -Eeuo pipefail + cargo codspeed run > codspeed-run.log + grep -E "^(Measured|Checked): " codspeed-run.log > codspeed-uris.txt + wc -l < codspeed-uris.txt + + - name: Upload measured benchmark URIs + if: needs.changes.outputs.run-budget-check == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: codspeed-uris-${{ matrix.shard }} + path: codspeed-uris.txt + retention-days: 1 + if-no-files-found: error + + # Enforces the "keep per-iteration execution time under ~1 ms" rule from + # docs/developer-guide/benchmarking.md. The simulation instrument above reports estimated + # cycles from an instruction trace, never wall-clock time, so the budget needs its own + # measurement: the same benchmarks rebuilt in walltime mode, where the divan harness dumps + # per-iteration statistics as JSON. Results are reported as a PR comment, not a hard + # failure. + bench-budget: + needs: [changes, bench-codspeed] + if: github.event_name == 'pull_request' && needs.changes.outputs.run-budget-check == 'true' + strategy: + matrix: + include: + - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error vortex-mask vortex-compute" } + - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" } + - { shard: 3, name: "Main library", packages: "vortex" } + - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" } + - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" } + - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" } + - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } + - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks vortex-row" } + name: "Check benchmark budget (Shard #${{ matrix.shard }})" + timeout-minutes: 30 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=bench-budget-{1}', github.run_id, matrix.shard) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + - name: Install Codspeed + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 + with: + tool: cargo-codspeed + - name: Download measured benchmark URIs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: codspeed-uris-${{ matrix.shard }} + - name: Build benchmarks + env: + RUSTFLAGS: "-C target-feature=+avx2" + run: cargo codspeed build -m walltime ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench + # `cargo codspeed run` only asks the harness to dump raw walltime JSON when it believes + # it is under the CodSpeed runner, so set CODSPEED_ENV ourselves. No token, no upload, + # and no valgrind: these numbers are for the budget check alone and are never reported + # to CodSpeed. Benchmarks that honour the budget make this run take milliseconds. + - name: Measure benchmarks + env: + CODSPEED_ENV: "1" + DIVAN_SAMPLE_COUNT: "3" + DIVAN_MIN_TIME: "0" + run: cargo codspeed run -m walltime + - name: Check against budget + run: | + python3 scripts/check-bench-budget.py check \ + --scope codspeed-uris.txt \ + --shard "${{ matrix.shard }}" \ + --output "verdicts/${{ matrix.shard }}.json" + - name: Upload verdict + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bench-budget-${{ matrix.shard }} + path: verdicts/ + retention-days: 1 + if-no-files-found: error + + bench-budget-comment: + needs: [bench-budget] + if: always() && needs.bench-budget.result == 'success' && github.event.pull_request.head.repo.fork == false + name: "Report benchmark budget" + timeout-minutes: 10 + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Download verdicts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: bench-budget-* + path: verdicts + - name: Render comment + run: | + python3 scripts/check-bench-budget.py report --inputs verdicts --output comment.md + cat comment.md >> "$GITHUB_STEP_SUMMARY" + - name: Comment PR + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + with: + file-path: comment.md + comment-tag: bench-budget-comment + # Getting a GPU box is slow, in the future we can build on a box without one and only run # on GPU machines. bench-codspeed-cuda-build: diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index 1655d7f8a08..86c77e680a7 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -124,6 +124,24 @@ fn my_bench(bencher: Bencher, num_indices: usize) { ... } Each individual iteration of the benchmarked closure should complete in **less than 1ms**. This is to keep benchmarks snappy, locally and on CI. +CI checks this on any pull request that touches a `benches/` directory, and reports the +result as a comment rather than failing the build. Because the simulation instrument +reports estimated cycles rather than wall-clock time, the `bench-budget` job in +`.github/workflows/codspeed.yml` rebuilds the same benchmarks in CodSpeed's walltime mode +and compares the *fastest* observed iteration against the budget -- the estimate least +affected by runner noise. Only benchmarks CodSpeed actually measures are checked, so +anything gated with `#[cfg(not(codspeed))]` is exempt automatically. + +To reproduce the check locally: + +```bash +cargo codspeed build -m walltime -p --profile bench +CODSPEED_ENV=1 cargo codspeed run -m walltime +python3 scripts/check-bench-budget.py check --shard local --output verdicts/local.json +``` + +Omitting `--scope` checks every benchmark, including the `#[cfg(not(codspeed))]` ones. + ### Gate CodSpeed-incompatible benchmarks Use `#[cfg(not(codspeed))]` for benchmarks that are incompatible with CodSpeed. diff --git a/scripts/check-bench-budget.py b/scripts/check-bench-budget.py new file mode 100644 index 00000000000..e3dbe212665 --- /dev/null +++ b/scripts/check-bench-budget.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Check that CodSpeed benchmarks stay under a per-iteration wall-clock budget. + +CodSpeed itself runs in ``simulation`` mode, which executes each benchmark exactly once +and estimates cycles from an instruction trace -- it never reports wall-clock time, so it +cannot enforce the "keep per-iteration execution time under ~1 ms" rule from +``docs/developer-guide/benchmarking.md``. + +The ``divan`` dependency is really ``codspeed-divan-compat``. Built *without* ``--cfg +codspeed`` it re-exports CodSpeed's patched divan, which writes one JSON file per +benchmark to ``target/codspeed/walltime/raw_results/divan/`` whenever ``CODSPEED_ENV`` is +set. Those files carry per-iteration statistics (the harness already divides each round by +its iteration count), which is exactly the quantity the budget is written against. + +Two subcommands: + +``check`` + Read the raw walltime results for one shard, restrict them to the benchmarks CodSpeed + actually measures, and emit a JSON verdict. + +``report`` + Merge the per-shard verdicts into a single Markdown comment body. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +DEFAULT_MAX_NS = 1_000_000 +"""One millisecond, per the benchmarking guide.""" + +GUIDE_URL = ( + "https://github.com/vortex-data/vortex/blob/develop/docs/developer-guide/benchmarking.md" + "#keep-per-iteration-execution-time-under-1-ms" +) + +MAX_ROWS = 30 +"""Cap the table so a wholesale regression cannot produce an unreadable comment.""" + + +def format_duration(nanos: float) -> str: + """Render a nanosecond count using the same units as divan's own output.""" + for limit, unit, scale in ( + (1_000, "ns", 1), + (1_000_000, "µs", 1_000), + (1_000_000_000, "ms", 1_000_000), + ): + if nanos < limit: + return f"{nanos / scale:.3g} {unit}" + return f"{nanos / 1_000_000_000:.3g} s" + + +def load_raw_results(raw_results: Path) -> list[dict]: + """Load every per-benchmark JSON file written by the patched divan harness.""" + benchmarks = [] + for path in sorted(raw_results.glob("**/*.json")): + with path.open() as f: + benchmarks.append(json.load(f)) + return benchmarks + + +def load_scope(scope: Path | None) -> set[str] | None: + """Load the set of benchmark URIs CodSpeed measures, or ``None`` for "all of them". + + The URIs are produced by the analysis-mode binaries, which print ``Measured: `` + (instrumented) or ``Checked: `` (not instrumented) for every benchmark they run. + Both modes build that URI with identical code, so the strings match the ``uri`` field + in the walltime results exactly. + """ + if scope is None: + return None + uris = set() + for line in scope.read_text().splitlines(): + line = line.strip() + for prefix in ("Measured: ", "Checked: "): + if line.startswith(prefix): + line = line[len(prefix) :] + break + # Instrumented runs append a group suffix that the URI itself does not carry. + line = line.split(" (group: ", 1)[0].strip() + if line: + uris.add(line) + return uris + + +def check(args: argparse.Namespace) -> int: + benchmarks = load_raw_results(args.raw_results) + scope = load_scope(args.scope) + + in_scope, violations = [], [] + for bench in benchmarks: + uri = bench["uri"] + if scope is not None and uri not in scope: + continue + in_scope.append(uri) + nanos = bench["stats"][args.metric] + if nanos > args.max_ns: + violations.append({"uri": uri, "name": bench["name"], "ns": nanos}) + + violations.sort(key=lambda v: v["ns"], reverse=True) + verdict = { + "shard": args.shard, + "max_ns": args.max_ns, + "metric": args.metric, + "measured": len(benchmarks), + "in_scope": len(in_scope), + "violations": violations, + } + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(verdict, indent=2) + "\n") + + print( + f"shard {args.shard}: {len(in_scope)}/{len(benchmarks)} benchmarks in scope, " + f"{len(violations)} over the {format_duration(args.max_ns)} budget" + ) + for violation in violations: + print(f" {format_duration(violation['ns']):>10} {violation['uri']}") + + return 1 if violations and args.fail_on_violation else 0 + + +def render_report(verdicts: list[dict]) -> str: + """Render the merged verdicts as the body of a sticky PR comment.""" + in_scope = sum(v["in_scope"] for v in verdicts) + violations = [v for verdict in verdicts for v in verdict["violations"]] + violations.sort(key=lambda v: v["ns"], reverse=True) + + # Every shard is configured identically; fall back to the default if none ran. + max_ns = verdicts[0]["max_ns"] if verdicts else DEFAULT_MAX_NS + metric = verdicts[0]["metric"] if verdicts else "min_ns" + budget = format_duration(max_ns) + + lines = ["## ⏱️ Benchmark iteration budget", ""] + + if not violations: + lines += [ + f"`✅ {in_scope}` CodSpeed benchmarks are within the **{budget}** " + "per-iteration budget.", + ] + else: + lines += [ + f"`⚠️ {len(violations)}` of `{in_scope}` CodSpeed benchmarks exceed the " + f"**{budget}** per-iteration budget.", + "", + "CodSpeed's simulation instrument runs each benchmark exactly once, so a slow " + "iteration costs CI time without buying any extra signal. Shrink the input " + f"size, or gate the benchmark with `#[cfg(not(codspeed))]`. See [the " + f"benchmarking guide]({GUIDE_URL}).", + "", + "| Benchmark | Fastest iteration | Over budget |", + "| --- | --- | --- |", + ] + for violation in violations[:MAX_ROWS]: + over = violation["ns"] / max_ns + lines.append( + f"| `{violation['uri']}` | {format_duration(violation['ns'])} | {over:.1f}× |" + ) + if len(violations) > MAX_ROWS: + lines += [ + "", + f"> ℹ️ _Only the first {MAX_ROWS} of {len(violations)} benchmarks are " + "displayed._", + ] + + lines += [ + "", + "
How this is measured", + "", + "Benchmarks are rebuilt in CodSpeed's walltime mode and run once outside the " + f"CodSpeed runner. The reported number is `{metric}` -- the fastest observed " + "iteration, which is the estimate least contaminated by runner noise, so a shared " + "CI machine cannot make this check flaky.", + "", + "Only benchmarks that CodSpeed actually measures are checked. That set comes from " + "the analysis-mode binaries built by `cargo codspeed build`, which enumerate every " + "benchmark they run, so anything behind `#[cfg(not(codspeed))]` is excluded " + "automatically rather than by an allowlist that can drift.", + "", + "
", + ] + return "\n".join(lines) + "\n" + + +def report(args: argparse.Namespace) -> int: + verdicts = [] + for path in sorted(args.inputs.glob("**/*.json")): + with path.open() as f: + verdicts.append(json.load(f)) + + if not verdicts: + print(f"no verdicts found under {args.inputs}", file=sys.stderr) + return 1 + + body = render_report(verdicts) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(body) + print(body) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + check_parser = subparsers.add_parser( + "check", help="check one shard's walltime results against the budget" + ) + check_parser.add_argument( + "--raw-results", + type=Path, + default=Path("target/codspeed/walltime/raw_results/divan"), + help="directory of per-benchmark JSON files written by the divan harness", + ) + check_parser.add_argument( + "--scope", + type=Path, + help="file listing the benchmark URIs CodSpeed measures; all are checked if omitted", + ) + check_parser.add_argument( + "--max-ns", + type=int, + default=DEFAULT_MAX_NS, + help=f"per-iteration budget in nanoseconds (default: {DEFAULT_MAX_NS})", + ) + check_parser.add_argument( + "--metric", + choices=["min_ns", "median_ns", "mean_ns", "max_ns"], + default="min_ns", + help="statistic to compare against the budget (default: min_ns)", + ) + check_parser.add_argument("--shard", default="", help="shard name, for reporting") + check_parser.add_argument( + "--output", type=Path, required=True, help="path to write the JSON verdict to" + ) + check_parser.add_argument( + "--fail-on-violation", + action="store_true", + help="exit non-zero when a benchmark is over budget (default: report only)", + ) + check_parser.set_defaults(func=check) + + report_parser = subparsers.add_parser( + "report", help="merge per-shard verdicts into a Markdown comment" + ) + report_parser.add_argument( + "--inputs", type=Path, required=True, help="directory of JSON verdicts" + ) + report_parser.add_argument( + "--output", type=Path, required=True, help="path to write the Markdown body to" + ) + report_parser.set_defaults(func=report) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_check_bench_budget.py b/scripts/tests/test_check_bench_budget.py new file mode 100644 index 00000000000..a38a2951012 --- /dev/null +++ b/scripts/tests/test_check_bench_budget.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +import importlib.util +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +BUDGET_SCRIPT = REPO_ROOT / "scripts" / "check-bench-budget.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("check_bench_budget", BUDGET_SCRIPT) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +budget = load_module() + + +def write_raw_result(directory: Path, uri: str, min_ns: float, name: str | None = None) -> None: + """Write a raw result in the shape the patched divan harness emits.""" + directory.mkdir(parents=True, exist_ok=True) + payload = { + "name": name or uri.rsplit("::", 1)[-1], + "uri": uri, + "config": {}, + "stats": { + "min_ns": min_ns, + "max_ns": min_ns * 1.2, + "mean_ns": min_ns * 1.1, + "median_ns": min_ns * 1.05, + "stdev_ns": 0.0, + "q1_ns": min_ns, + "q3_ns": min_ns * 1.1, + "rounds": 3, + "total_time": 0.1, + "iqr_outlier_rounds": 0, + "stdev_outlier_rounds": 0, + "iter_per_round": 1, + "warmup_iters": 0, + }, + } + (directory / f"{abs(hash(uri))}.json").write_text(json.dumps(payload)) + + +def run_check(tmp_path: Path, **overrides): + raw = tmp_path / "raw" + output = tmp_path / "verdict.json" + args = { + "raw_results": raw, + "scope": None, + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "shard": "1", + "output": output, + "fail_on_violation": False, + } + args.update(overrides) + code = budget.check(_Namespace(**args)) + return code, json.loads(output.read_text()) + + +class _Namespace: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +@pytest.mark.parametrize( + ("nanos", "expected"), + [(36, "36 ns"), (61_600, "61.6 µs"), (14_100_000, "14.1 ms"), (2_000_000_000, "2 s")], +) +def test_format_duration_matches_divan_units(nanos, expected): + assert budget.format_duration(nanos) == expected + + +def test_check_flags_only_over_budget_benchmarks(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "vortex-mask/benches/rank.rs::fast", 36) + write_raw_result(raw, "vortex-geo/benches/envelope.rs::slow", 65_500_000) + + code, verdict = run_check(tmp_path) + + assert code == 0 + assert verdict["in_scope"] == 2 + assert [v["uri"] for v in verdict["violations"]] == ["vortex-geo/benches/envelope.rs::slow"] + + +def test_check_ignores_benchmarks_codspeed_does_not_measure(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "vortex/benches/throughput.rs::gated", 65_500_000) + write_raw_result(raw, "vortex-mask/benches/rank.rs::fast", 36) + scope = tmp_path / "scope.txt" + scope.write_text("Measured: vortex-mask/benches/rank.rs::fast\n") + + code, verdict = run_check(tmp_path, scope=scope) + + assert code == 0 + assert verdict["measured"] == 2 + assert verdict["in_scope"] == 1 + assert verdict["violations"] == [] + + +def test_check_can_fail_the_job(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "vortex-geo/benches/envelope.rs::slow", 65_500_000) + + code, _ = run_check(tmp_path, fail_on_violation=True) + + assert code == 1 + + +def test_check_orders_violations_worst_first(tmp_path): + raw = tmp_path / "raw" + write_raw_result(raw, "a.rs::mid", 14_100_000) + write_raw_result(raw, "a.rs::worst", 2_000_000_000) + write_raw_result(raw, "a.rs::least", 1_000_001) + + _, verdict = run_check(tmp_path) + + assert [v["uri"] for v in verdict["violations"]] == ["a.rs::worst", "a.rs::mid", "a.rs::least"] + + +@pytest.mark.parametrize( + "line", + [ + "Measured: vortex-mask/benches/rank.rs::fast", + "Checked: vortex-mask/benches/rank.rs::fast", + "Measured: vortex-mask/benches/rank.rs::fast (group: outer/inner)", + " Measured: vortex-mask/benches/rank.rs::fast ", + ], +) +def test_load_scope_parses_harness_output(tmp_path, line): + scope = tmp_path / "scope.txt" + scope.write_text(f"{line}\n") + + assert budget.load_scope(scope) == {"vortex-mask/benches/rank.rs::fast"} + + +def test_load_scope_of_none_means_check_everything(): + assert budget.load_scope(None) is None + + +def test_report_merges_shards_and_sorts_globally(): + body = budget.render_report( + [ + { + "shard": "1", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 2, + "in_scope": 2, + "violations": [{"uri": "a.rs::mid", "name": "mid", "ns": 14_100_000}], + }, + { + "shard": "2", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 3, + "in_scope": 3, + "violations": [{"uri": "b.rs::worst", "name": "worst", "ns": 2_000_000_000}], + }, + ] + ) + + assert "`⚠️ 2` of `5` CodSpeed benchmarks exceed" in body + assert body.index("b.rs::worst") < body.index("a.rs::mid") + assert "| `b.rs::worst` | 2 s | 2000.0× |" in body + + +def test_report_is_reassuring_when_clean(): + body = budget.render_report( + [ + { + "shard": "1", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 7, + "in_scope": 7, + "violations": [], + } + ] + ) + + assert "`✅ 7` CodSpeed benchmarks are within the **1 ms** per-iteration budget." in body + assert "| Benchmark |" not in body + + +def test_report_truncates_a_wholesale_regression(): + violations = [{"uri": f"a.rs::b{i}", "name": f"b{i}", "ns": 2_000_000 + i} for i in range(50)] + body = budget.render_report( + [ + { + "shard": "1", + "max_ns": budget.DEFAULT_MAX_NS, + "metric": "min_ns", + "measured": 50, + "in_scope": 50, + "violations": violations, + } + ] + ) + + assert body.count("| `a.rs::") == budget.MAX_ROWS + assert f"Only the first {budget.MAX_ROWS} of 50 benchmarks are displayed." in body diff --git a/vortex-mask/Cargo.toml b/vortex-mask/Cargo.toml index cf681101d43..2076f56586b 100644 --- a/vortex-mask/Cargo.toml +++ b/vortex-mask/Cargo.toml @@ -45,5 +45,10 @@ harness = false name = "mask_iteration" harness = false +# TEMPORARY: proves the bench-budget CI check fires. Delete before merge. +[[bench]] +name = "budget_canary" +harness = false + [lints] workspace = true diff --git a/vortex-mask/benches/budget_canary.rs b/vortex-mask/benches/budget_canary.rs new file mode 100644 index 00000000000..94713e0987e --- /dev/null +++ b/vortex-mask/benches/budget_canary.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! TEMPORARY: a deliberately over-budget benchmark, used to prove that the `bench-budget` +//! CI job actually detects and reports a violation. +//! +//! This exists only to exercise the check on the pull request that introduces it, and +//! **must be deleted before merge**. It intentionally violates the "keep per-iteration +//! execution time under ~1 ms" rule in `docs/developer-guide/benchmarking.md`. +//! +//! The work is real rather than a `sleep`, because CodSpeed's simulation instrument +//! excludes system calls -- a sleeping benchmark would look free there while still +//! burning CI wall-clock time. + +#![allow(clippy::unwrap_used, clippy::cast_possible_truncation)] + +use divan::Bencher; +use vortex_buffer::BitBuffer; +use vortex_mask::Mask; + +fn main() { + divan::main(); +} + +/// 256Mi bits is a 32 MiB bit buffer. Counting the set bits in a prefix of it is bound by +/// memory bandwidth, which puts a single iteration well past the 1 ms budget on any runner. +const CANARY_LEN: usize = 256 * 1024 * 1024; + +#[divan::bench] +fn over_budget_prefix_count(bencher: Bencher) { + let mask = Mask::from_buffer(BitBuffer::from_iter( + (0..CANARY_LEN).map(|i| (i * 7 + 13) % 1000 < 900), + )); + let indices = [CANARY_LEN / 4, CANARY_LEN - CANARY_LEN / 8]; + bencher + .with_inputs(|| (&mask, indices)) + .bench_refs(|(mask, indices)| mask.valid_counts_for_indices(indices)); +}