feat(bench): 构建引擎基准套件 —— 把一次性脚本变成跨平台、可扩展的测量设施 (2026.8.12.1) #43
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: bench | |
| # Build-engine benchmark. Runs on changes under `bench/` and on demand. | |
| # | |
| # WHY IT IS PATH-SCOPED RATHER THAN ON EVERY PUSH: | |
| # | |
| # * it is heavy — a full matrix compiles the same fixture six ways per platform | |
| # * it is noisy — cloud runners are shared, and the CPU model changes under you | |
| # * it asserts nothing about TIMINGS — no threshold, no pass/fail on seconds | |
| # | |
| # So it fires when the SUITE itself changes, where the question "did I break the | |
| # harness / did this shift the numbers" is actually being asked, and stays off | |
| # every unrelated PR. A timing threshold on a shared runner would turn normal | |
| # variance into red crosses people learn to ignore, so there is none: results | |
| # are uploaded as artifacts and comparing them is a human act. | |
| # | |
| # IT DOES ASSERT THAT SOMETHING WAS MEASURED. The harness exits non-zero when a | |
| # cell `failed` — the engine ran and produced no artifact — or when nothing was | |
| # measured at all. That is not a timing threshold, and its absence was expensive: | |
| # a "passing" matrix job had 6 ok / 48 failed / 18 unavailable, and every xlings | |
| # job had zero measurements, for weeks. | |
| # | |
| # The matrix runs platforms in parallel and `fail-fast: false`, because one | |
| # platform missing an engine must not cancel the data from the others. | |
| # | |
| # See bench/README.md for the measurement contract before quoting any number. | |
| on: | |
| # Changes that can MOVE THE NUMBERS: the harness itself, the build | |
| # descriptions of the projects it measures, its own tests, and the PINNED | |
| # TREES it measures (a submodule bump is a different benchmark target). | |
| # | |
| # Documentation and past results are excluded on purpose. A README edit cannot | |
| # change a measurement, and running a two-hour matrix to prove that teaches | |
| # everyone to ignore the check — which is how a benchmark stops being read. | |
| # `bench/results/**` is excluded for the same reason AND a sharper one: this | |
| # workflow's own artifacts land there, so including it would let a results | |
| # commit trigger the run that produces the next results commit. | |
| push: | |
| paths: | |
| - 'bench/**' | |
| - '!bench/**/*.md' | |
| - '!bench/results/**' | |
| - '.gitmodules' | |
| - '.github/workflows/bench.yml' | |
| pull_request: | |
| paths: | |
| - 'bench/**' | |
| - '!bench/**/*.md' | |
| - '!bench/results/**' | |
| - '.gitmodules' | |
| - '.github/workflows/bench.yml' | |
| workflow_dispatch: | |
| # These FILTER the cell list in bench/matrix.json; they do not replace it. | |
| # Which engines / variants / scenarios a cell sweeps is a property of the | |
| # cell (a gcc cell cannot run bazel's module support, a real project has no | |
| # `headers` form), so those live in matrix.json next to the cell they | |
| # describe rather than as one global default applied to every platform. | |
| inputs: | |
| preset: | |
| description: 'named fixture size: smoke | standard | large (overridden by units/fanin/weight below)' | |
| required: false | |
| default: 'standard' | |
| units: | |
| description: 'fixture translation units (0 = use the preset)' | |
| required: false | |
| default: '0' | |
| fanin: | |
| description: 'dependencies per unit (controls graph depth)' | |
| required: false | |
| default: '3' | |
| runs: | |
| description: 'repetitions per cell (0 = per-scenario default)' | |
| required: false | |
| default: '0' | |
| profile: | |
| description: 'release | debug' | |
| required: false | |
| default: 'release' | |
| platforms: | |
| description: 'FILTER on bench/matrix.json cells: linux,macos,windows' | |
| required: false | |
| default: 'linux,macos,windows' | |
| toolchains: | |
| description: 'FILTER on bench/matrix.json cells: gcc,clang,msvc' | |
| required: false | |
| default: 'gcc,clang,msvc' | |
| projects: | |
| description: 'FILTER on bench/matrix.json cells (substring match): fixture,mcpp,xlings' | |
| required: false | |
| default: 'fixture,mcpp,xlings' | |
| concurrency: | |
| group: bench-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # The matrix is READ, not written here. bench/matrix.json is the single source | |
| # of truth for which (OS, toolchain, project) cells exist, which engines / | |
| # variants / scenarios each one sweeps, WHICH TOOL VERSIONS are installed and | |
| # which file each scenario perturbs; bench/SPEC.md explains the axes and | |
| # deliberately does not repeat the list. A matrix written down twice is a | |
| # matrix that disagrees with itself, and the disagreement is silent — both | |
| # copies keep looking right. | |
| # | |
| # The dispatch inputs FILTER that list rather than replace it, so | |
| # `platforms: linux` runs the linux cells and nothing else — a skipped job | |
| # still queues a runner and still reports a check. | |
| plan: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| tools: ${{ steps.plan.outputs.tools }} | |
| reference_mcpp: ${{ steps.plan.outputs.reference_mcpp }} | |
| baseline: ${{ steps.plan.outputs.baseline }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: plan | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # `inputs.*` is empty on push/pull_request, so every one needs a | |
| # fallback here — an empty filter would otherwise plan an empty matrix | |
| # and the job would silently do nothing. | |
| plat="${{ inputs.platforms || 'linux,macos,windows' }}" | |
| tool="${{ inputs.toolchains || 'gcc,clang,msvc' }}" | |
| proj="${{ inputs.projects || 'fixture,mcpp,xlings' }}" | |
| # `. as $c` is load-bearing: inside `$plat | contains(...)` the `.` has | |
| # already become $plat, so a bare `.os` there indexes a STRING and jq | |
| # fails pointing at a line number in the data file rather than at the | |
| # program. | |
| # | |
| # The PROJECT filter is a prefix match, not equality: the project axis | |
| # carries pinned versions (`xlings-2026.8.13.1`), and a dispatch asking | |
| # for `xlings` means "both styles" rather than "nothing". | |
| include=$(jq -c \ | |
| --arg plat ",$plat," --arg tool ",$tool," --arg proj "$proj" ' | |
| [ .cells[] | |
| | . as $c | |
| | select($plat | contains("," + $c.os + ",")) | |
| | select($tool | contains("," + $c.toolchain + ",")) | |
| | select($proj | split(",") | any(. as $p | $c.project | startswith($p))) | |
| | $c | |
| # `buildfiles` defaults to the project name; only the versioned | |
| # xlings trees need it, since several of them share one description. | |
| + { buildfiles: ($c.buildfiles // $c.project) } | |
| + { runs_on: $runners[$c.os] } | |
| ]' --argjson runners "$(jq -c .runners bench/matrix.json)" bench/matrix.json) | |
| count=$(printf '%s' "$include" | jq 'length') | |
| if [ "$count" -eq 0 ]; then | |
| echo "no cell in bench/matrix.json matches platforms='$plat' toolchains='$tool' projects='$proj'" >&2 | |
| exit 1 | |
| fi | |
| echo "planning $count cell(s):" | |
| printf '%s' "$include" | jq -r '.[] | " \(.os)/\(.toolchain)/\(.project)"' | |
| printf 'matrix={"include":%s}\n' "$include" >> "$GITHUB_OUTPUT" | |
| # The tool pins travel with the plan so every job installs the same | |
| # versions from one declaration. See matrix.json's `tools._note` for | |
| # what an unpinned matrix was actually measuring. | |
| printf 'tools=%s\n' "$(jq -c .tools bench/matrix.json)" >> "$GITHUB_OUTPUT" | |
| printf 'reference_mcpp=%s\n' "$(jq -r .reference_mcpp bench/matrix.json)" >> "$GITHUB_OUTPUT" | |
| printf 'baseline=%s\n' "$(jq -r .baseline bench/matrix.json)" >> "$GITHUB_OUTPUT" | |
| echo "tool pins: $(jq -c '.tools | del(._note, ._compiler_note)' bench/matrix.json)" | |
| bench: | |
| needs: plan | |
| strategy: | |
| fail-fast: false # one platform's engine gap must not cancel the rest | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| runs-on: ${{ matrix.runs_on }} | |
| timeout-minutes: 120 | |
| name: bench (${{ matrix.os }}/${{ matrix.toolchain }}/${{ matrix.project }}) | |
| env: | |
| TOOLS: ${{ needs.plan.outputs.tools }} | |
| REFERENCE_MCPP: ${{ needs.plan.outputs.reference_mcpp }} | |
| steps: | |
| # submodules: the pinned trees under bench/projects/ ARE the xlings | |
| # projects being measured. They used to be cloned at run time from the | |
| # default branch, which meant the benchmark target moved with every | |
| # upstream push — and it had already moved out from under `--hub`. | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: ./.github/actions/bootstrap-mcpp | |
| # ── Every tool at a pinned version, all of them through xlings ───────── | |
| # | |
| # Not best-effort any more. An engine that is absent is reported as | |
| # `unavailable` and is fine; an engine present at the WRONG VERSION is not | |
| # fine and is invisible — the runner image's cmake 3.31.6 cannot configure | |
| # C++23 modules, so it turned every module cell into `failed` while the | |
| # job stayed green. | |
| - name: Install the pinned build tools via xlings | |
| shell: bash | |
| run: | | |
| set -uo pipefail | |
| for t in cmake xmake bazel; do | |
| v=$(printf '%s' "$TOOLS" | jq -r --arg t "$t" '.[$t]') | |
| echo "::group::xlings install $t@$v" | |
| xlings install "$t@$v" -y || echo "::warning::$t@$v is not installable on this runner; it will report as unavailable" | |
| echo "::endgroup::" | |
| done | |
| # The last RELEASED mcpp, so every report carries an old-vs-new column | |
| # rather than only saying how fast this branch is. | |
| echo "::group::xlings install mcpp@$REFERENCE_MCPP" | |
| xlings install "mcpp@$REFERENCE_MCPP" -y || echo "::warning::mcpp@$REFERENCE_MCPP unavailable; the reference column will be missing" | |
| echo "::endgroup::" | |
| # Loud, because a version that quietly differs from the pin is the whole | |
| # class of bug this section exists to end. | |
| - name: Report the resolved tool versions | |
| shell: bash | |
| run: | | |
| set -uo pipefail | |
| fail=0 | |
| check() { # check <tool> <pinned> <actual-version-string> | |
| case "$3" in | |
| *"$2"*) echo " $1 $3 (pinned $2)" ;; | |
| *) echo "::warning::$1 resolved to '$3' but matrix.json pins $2 — this cell measures a different tool than it claims"; fail=1 ;; | |
| esac | |
| } | |
| check cmake "$(printf '%s' "$TOOLS" | jq -r .cmake)" "$(cmake --version 2>/dev/null | head -1)" | |
| check xmake "$(printf '%s' "$TOOLS" | jq -r .xmake)" "$(xmake --version 2>/dev/null | head -1)" | |
| check bazel "$(printf '%s' "$TOOLS" | jq -r .bazel)" "$(bazel --version 2>/dev/null | head -1)" | |
| ninja --version || true | |
| exit 0 | |
| # ── The two mcpp binaries being compared ─────────────────────────────── | |
| - name: Build the mcpp under test | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| "$MCPP" build --release | |
| # NEWEST, not `head -1`: target/ holds one directory per toolchain | |
| # fingerprint and a plain `find | head -1` picks whichever the | |
| # filesystem lists first, which is routinely a stale binary from an | |
| # earlier fingerprint. This benchmark would then measure the wrong mcpp | |
| # and say nothing. | |
| BIN=$(bash .github/tools/newest_artifact.sh target 'mcpp') | |
| echo "MCPP_UNDER_TEST=$BIN" >> "$GITHUB_ENV" | |
| echo "under test : $("$BIN" --version)" | |
| echo "reference : $(command -v mcpp && mcpp --version || echo 'not installed')" | |
| - name: Build the harness | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd bench | |
| "$MCPP" build --release | |
| BIN=$(bash ../.github/tools/newest_artifact.sh target 'bench') | |
| echo "BENCH=$PWD/$BIN" >> "$GITHUB_ENV" | |
| # The compiler axis. Resolved to MCPP'S OWN PAYLOAD driver, not to | |
| # `command -v g++`, because those are not the same compiler: the runner's | |
| # gcc is 13.3.0, which cmake cannot configure C++23 modules with and which | |
| # xmake crashes outright, while mcpp silently used the registry's gcc 16.1 | |
| # anyway. The suite's fairness rule is that every engine gets the SAME | |
| # compiler; `payload:` is how the harness delivers it. msvc is the | |
| # exception: cl.exe is reached through the VS environment, not a path. | |
| - name: Resolve the compiler for this cell | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # The payload has to BE THERE. `mcpp build` installs whatever the | |
| # manifest's default toolchain is and nothing else, so on a clang cell | |
| # the llvm payload may simply not exist — and `--compiler payload:clang` | |
| # is a hard error by design rather than a silent fall back to the host | |
| # clang, which is the failure mode this whole arrangement exists to | |
| # prevent. Install it explicitly, once, before it is asked for. | |
| case "${{ matrix.toolchain }}" in | |
| msvc) | |
| # No payload: mcpp uses the system Visual Studio (`msvc@system`), | |
| # reached through the VS environment rather than through a path. | |
| echo "BENCH_CXX=msvc" >> "$GITHUB_ENV" ;; | |
| gcc) | |
| "$MCPP" toolchain install "gcc@$(printf '%s' "$TOOLS" | jq -r .gcc)" | |
| echo "BENCH_CXX=payload:gcc" >> "$GITHUB_ENV" ;; | |
| clang) | |
| # Windows is pinned to a different llvm than the other platforms — | |
| # see bench/src/toolchain.cppm, which is where the harness looks it | |
| # up, so the two must name the same version. | |
| if [ "${{ matrix.os }}" = "windows" ]; then key=.llvm_windows; else key=.llvm; fi | |
| "$MCPP" toolchain install "llvm@$(printf '%s' "$TOOLS" | jq -r "$key")" | |
| echo "BENCH_CXX=payload:clang" >> "$GITHUB_ENV" ;; | |
| esac | |
| "$MCPP" toolchain list || true | |
| echo "cell compiler: ${{ matrix.toolchain }}" | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| if: matrix.toolchain == 'msvc' | |
| # The project axis. `fixture` needs nothing — the harness generates it. | |
| # EVERY other workload is a PINNED SUBMODULE under bench/projects/, which | |
| # is why there is no clone step here any more and no special case for | |
| # mcpp's own sources. | |
| - name: Locate the project under measurement | |
| if: matrix.project != 'fixture' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # One rule, no special cases: every workload is a pinned submodule. | |
| # mcpp's own sources used to be `$GITHUB_WORKSPACE`, which made the | |
| # thing being measured change with every commit on the branch. | |
| root="$GITHUB_WORKSPACE/bench/projects/${{ matrix.buildfiles }}/${{ matrix.project }}" | |
| [ -e "$root/mcpp.toml" ] || { | |
| echo "no project at $root — is the submodule checked out?" >&2 | |
| ls -la "$(dirname "$root")" >&2 || true | |
| exit 1 | |
| } | |
| echo "BENCH_PROJECT=$root" >> "$GITHUB_ENV" | |
| git -C "$root" rev-parse HEAD 2>/dev/null || true | |
| - name: Report engine availability | |
| shell: bash | |
| run: | | |
| "$BENCH" --list | |
| - name: Run benchmark | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # `mcpp` in the cell's engine list means BOTH mcpp binaries: the one | |
| # built from this checkout and the last released one. Each labels | |
| # itself from the version it reports, so the rows stay distinct. | |
| # | |
| # Split on commas rather than sed: `\b` is a GNU extension that BSD sed | |
| # (i.e. every macOS runner) does not implement, and it fails by NOT | |
| # substituting — the macOS cells would quietly measure one mcpp while | |
| # the Linux ones measured two. | |
| # THREE arms, not two. The third is the same binary with the BMI | |
| # schedule turned on (`[build] bmi_schedule = "on"`), which is opt-in | |
| # until it is verified on every platform — and which the suite could | |
| # not otherwise reach at all, because the switch lives in the MEASURED | |
| # PROJECT's manifest and the measured projects are pinned workloads | |
| # that are not ours to edit. Without it the table said "no improvement | |
| # on cold builds" for the largest cold-build change in the release. | |
| engines="" | |
| IFS=',' read -ra want <<< '${{ matrix.engines }}' | |
| for e in "${want[@]}"; do | |
| if [ "$e" = "mcpp" ]; then | |
| e="mcpp=$MCPP_UNDER_TEST,mcpp[schedule=on]=$MCPP_UNDER_TEST,mcpp" | |
| fi | |
| engines="${engines:+$engines,}$e" | |
| done | |
| echo "engines: $engines" | |
| # --timeout: one configure/build may take 30 minutes. Two jobs once sat | |
| # 25 minutes inside a single hung child with a completely silent log. | |
| # The baseline is per-cell with a global default. The xlings arms | |
| # override it to the released mcpp because their cmake/xmake arms stop | |
| # at the link, and normalising against an engine that never produced a | |
| # binary prints bare seconds under a heading that says "relative to". | |
| args=( --engines "$engines" | |
| --variants '${{ matrix.variants }}' | |
| --scenarios '${{ matrix.scenarios }}' | |
| --baseline '${{ matrix.baseline || needs.plan.outputs.baseline }}' | |
| --profile '${{ inputs.profile || 'release' }}' | |
| --runs '${{ inputs.runs || 0 }}' | |
| --timeout 1800 | |
| --work "$RUNNER_TEMP/bench-work" | |
| --out "bench-${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.project }}.json" ) | |
| # `msvc` is a label, not a path — see the resolve step above. | |
| [ "$BENCH_CXX" != "msvc" ] && [ -n "$BENCH_CXX" ] && args+=( --compiler "$BENCH_CXX" ) | |
| if [ "${{ matrix.project }}" = "fixture" ]; then | |
| # The preset names the size; units/fanin override it only when set to | |
| # a positive number. Passing raw numbers unconditionally would make | |
| # every run's size an accident of this file rather than a named, | |
| # comparable workload — and --preset must come first so the | |
| # overrides still win. | |
| args+=( --preset "${{ inputs.preset || matrix.preset }}" ) | |
| [ "${{ inputs.units || 0 }}" -gt 0 ] 2>/dev/null && args+=( --units "${{ inputs.units }}" ) | |
| [ "${{ inputs.fanin || 0 }}" -gt 0 ] 2>/dev/null && args+=( --fanin "${{ inputs.fanin }}" ) | |
| else | |
| # A real tree: measured in place, and the scenarios that perturb a | |
| # file must be TOLD which one — from matrix.json, next to the cell, | |
| # rather than from a `case` here. The previous copy in this file | |
| # named `src/xlings.cppm`, which had not existed for months. | |
| args+=( --project "$BENCH_PROJECT" | |
| --buildfiles "$GITHUB_WORKSPACE/bench/projects/${{ matrix.buildfiles }}" | |
| --hub '${{ matrix.hub }}' | |
| --body '${{ matrix.body }}' ) | |
| [ -n '${{ matrix.allow_failed }}' ] && args+=( --allow-failed '${{ matrix.allow_failed }}' ) | |
| fi | |
| "$BENCH" "${args[@]}" | |
| # if: always() — a failed cell still produced a report, and that report is | |
| # the evidence for WHY it failed. Uploading only on success would throw | |
| # away the run worth looking at. | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.project }} | |
| path: bench-${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.project }}.json | |
| if-no-files-found: warn |