refactor(bench): 移除 meson,补齐 xlings 的 xmake/bazel,工具链逻辑收进 common/ #31
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: bench | |
| # Build-engine benchmark. Runs on changes under `bench/` and on demand. | |
| # | |
| # WHY IT IS PATH-SCOPED RATHER THAN ON EVERY PUSH: | |
| # | |
| # * it is heavy — a full matrix compiles the same fixture six ways per platform | |
| # * it is noisy — cloud runners are shared, and the CPU model changes under you | |
| # * it asserts nothing — no threshold, no pass/fail on timings | |
| # | |
| # So it fires when the SUITE itself changes, where the question "did I break the | |
| # harness / did this shift the numbers" is actually being asked, and stays off | |
| # every unrelated PR. A timing threshold on a shared runner would turn normal | |
| # variance into red crosses people learn to ignore, so there is none: results | |
| # are uploaded as artifacts and comparing them is a human act. | |
| # | |
| # The matrix runs platforms in parallel and `fail-fast: false`, because one | |
| # platform missing an engine must not cancel the data from the others. | |
| # | |
| # See bench/README.md for the measurement contract before quoting any number. | |
| on: | |
| # Changes that can MOVE THE NUMBERS: the harness itself, the build | |
| # descriptions of the projects it measures, and its own tests. | |
| # | |
| # Documentation and past results are excluded on purpose. A README edit cannot | |
| # change a measurement, and running a two-hour matrix to prove that teaches | |
| # everyone to ignore the check — which is how a benchmark stops being read. | |
| # `bench/results/**` is excluded for the same reason AND a sharper one: this | |
| # workflow's own artifacts land there, so including it would let a results | |
| # commit trigger the run that produces the next results commit. | |
| push: | |
| paths: | |
| - 'bench/**' | |
| - '!bench/**/*.md' | |
| - '!bench/results/**' | |
| - '.github/workflows/bench.yml' | |
| pull_request: | |
| paths: | |
| - 'bench/**' | |
| - '!bench/**/*.md' | |
| - '!bench/results/**' | |
| - '.github/workflows/bench.yml' | |
| workflow_dispatch: | |
| # These FILTER the cell list in bench/matrix.json; they do not replace it. | |
| # Which engines / variants / scenarios a cell sweeps is a property of the | |
| # cell (a gcc cell cannot run bazel's module support, a real project has no | |
| # `headers` form), so those live in matrix.json next to the cell they | |
| # describe rather than as one global default applied to every platform. | |
| inputs: | |
| preset: | |
| description: 'named fixture size: smoke | standard | large (overridden by units/fanin/weight below)' | |
| required: false | |
| default: 'standard' | |
| units: | |
| description: 'fixture translation units (0 = use the preset)' | |
| required: false | |
| default: '0' | |
| fanin: | |
| description: 'dependencies per unit (controls graph depth)' | |
| required: false | |
| default: '3' | |
| runs: | |
| description: 'repetitions per cell (0 = per-scenario default)' | |
| required: false | |
| default: '0' | |
| profile: | |
| description: 'release | debug' | |
| required: false | |
| default: 'release' | |
| platforms: | |
| description: 'FILTER on bench/matrix.json cells: linux,macos,windows' | |
| required: false | |
| default: 'linux,macos,windows' | |
| toolchains: | |
| description: 'FILTER on bench/matrix.json cells: gcc,clang,msvc' | |
| required: false | |
| default: 'gcc,clang,msvc' | |
| projects: | |
| description: 'FILTER on bench/matrix.json cells: fixture,mcpp,xlings' | |
| required: false | |
| default: 'fixture,mcpp,xlings' | |
| concurrency: | |
| group: bench-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # The matrix is READ, not written here. bench/matrix.json is the single source | |
| # of truth for which (OS, toolchain, project) cells exist and which engines / | |
| # variants / scenarios each one sweeps; bench/SPEC.md explains the axes and | |
| # deliberately does not repeat the list. A matrix written down twice is a | |
| # matrix that disagrees with itself, and the disagreement is silent — both | |
| # copies keep looking right. | |
| # | |
| # The dispatch inputs FILTER that list rather than replace it, so | |
| # `platforms: linux` runs the linux cells and nothing else — a skipped job | |
| # still queues a runner and still reports a check. | |
| plan: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: plan | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # `inputs.*` is empty on push/pull_request, so every one needs a | |
| # fallback here — an empty filter would otherwise plan an empty matrix | |
| # and the job would silently do nothing. | |
| plat="${{ inputs.platforms || 'linux,macos,windows' }}" | |
| tool="${{ inputs.toolchains || 'gcc,clang,msvc' }}" | |
| proj="${{ inputs.projects || 'fixture,mcpp,xlings' }}" | |
| # `. as $c` is load-bearing: inside `$plat | contains(...)` the `.` has | |
| # already become $plat, so a bare `.os` there indexes a STRING and jq | |
| # fails pointing at a line number in the data file rather than at the | |
| # program. | |
| include=$(jq -c \ | |
| --arg plat ",$plat," --arg tool ",$tool," --arg proj ",$proj," \ | |
| --argjson runners "$(jq -c .runners bench/matrix.json)" ' | |
| [ .cells[] | |
| | . as $c | |
| | select($plat | contains("," + $c.os + ",")) | |
| | select($tool | contains("," + $c.toolchain + ",")) | |
| | select($proj | contains("," + $c.project + ",")) | |
| | $c + { runs_on: $runners[$c.os] } | |
| ]' bench/matrix.json) | |
| count=$(printf '%s' "$include" | jq 'length') | |
| if [ "$count" -eq 0 ]; then | |
| echo "no cell in bench/matrix.json matches platforms='$plat' toolchains='$tool' projects='$proj'" >&2 | |
| exit 1 | |
| fi | |
| echo "planning $count cell(s):" | |
| printf '%s' "$include" | jq -r '.[] | " \(.os)/\(.toolchain)/\(.project)"' | |
| printf 'matrix={"include":%s}\n' "$include" >> "$GITHUB_OUTPUT" | |
| bench: | |
| needs: plan | |
| strategy: | |
| fail-fast: false # one platform's engine gap must not cancel the rest | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| runs-on: ${{ matrix.runs_on }} | |
| timeout-minutes: 120 | |
| name: bench (${{ matrix.os }}/${{ matrix.toolchain }}/${{ matrix.project }}) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/bootstrap-mcpp | |
| - name: Build the harness | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd bench | |
| "$MCPP" build --release | |
| # Resolve the produced binary once; the fingerprint directory name is | |
| # not predictable from here. | |
| BIN=$(find target -type f -name 'bench' -o -type f -name 'bench.exe' | head -1) | |
| [ -n "$BIN" ] || { echo "harness binary not found under bench/target" >&2; exit 1; } | |
| echo "BENCH=$PWD/$BIN" >> "$GITHUB_ENV" | |
| # Engines beyond mcpp are optional by design: a missing one is reported as | |
| # `unavailable` with a reason, never as a slow or broken engine. Installing | |
| # them is therefore best-effort and never fails the job. | |
| - name: Install comparison engines (best effort) | |
| shell: bash | |
| continue-on-error: true | |
| run: | | |
| set -uo pipefail | |
| xlings install bazel -y || echo "bazel unavailable on this runner" | |
| xlings install xmake -y || echo "xmake unavailable on this runner" | |
| cmake --version || true | |
| ninja --version || true | |
| # The compiler axis. Resolved to a DRIVER PATH here rather than passed as a | |
| # label, because `--compiler clang` means "whatever clang++ is on PATH" and | |
| # that is a different compiler on each runner — which is exactly the | |
| # comparison this suite is not making. msvc is the exception: cl.exe is | |
| # reached through the VS environment, not a path, so the label is passed | |
| # through and each engine's msvc handling applies. | |
| - name: Resolve the compiler for this cell | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| case "${{ matrix.toolchain }}" in | |
| msvc) echo "BENCH_CXX=msvc" >> "$GITHUB_ENV" ;; | |
| gcc) echo "BENCH_CXX=$(command -v g++)" >> "$GITHUB_ENV" ;; | |
| clang) echo "BENCH_CXX=$(command -v clang++)" >> "$GITHUB_ENV" ;; | |
| esac | |
| echo "cell compiler: ${{ matrix.toolchain }}" | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| if: matrix.toolchain == 'msvc' | |
| # The project axis. `fixture` needs nothing — the harness generates it. | |
| # `xlings` is cloned rather than vendored: a vendored snapshot rots, and a | |
| # benchmark whose target drifts from the real project measures the | |
| # snapshot (bench/projects/xlings/README.md). | |
| - name: Fetch the project under measurement | |
| if: matrix.project == 'xlings' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| git clone --depth 1 https://github.com/openxlings/xlings "$RUNNER_TEMP/xlings" | |
| echo "BENCH_PROJECT=$RUNNER_TEMP/xlings" >> "$GITHUB_ENV" | |
| git -C "$RUNNER_TEMP/xlings" rev-parse HEAD | |
| - name: Locate the project under measurement | |
| if: matrix.project == 'mcpp' | |
| shell: bash | |
| run: echo "BENCH_PROJECT=$GITHUB_WORKSPACE" >> "$GITHUB_ENV" | |
| - name: Report engine availability | |
| shell: bash | |
| run: | | |
| "$BENCH" --list | |
| - name: Run benchmark | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| args=( --engines '${{ matrix.engines }}' | |
| --variants '${{ matrix.variants }}' | |
| --scenarios '${{ matrix.scenarios }}' | |
| --baseline cmake | |
| --profile '${{ inputs.profile || 'release' }}' | |
| --runs '${{ inputs.runs || 0 }}' | |
| --work "$RUNNER_TEMP/bench-work" | |
| --out "bench-${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.project }}.json" ) | |
| # `msvc` is a label, not a path — see the resolve step above. | |
| [ "$BENCH_CXX" != "msvc" ] && [ -n "$BENCH_CXX" ] && args+=( --compiler "$BENCH_CXX" ) | |
| if [ "${{ matrix.project }}" = "fixture" ]; then | |
| # The preset names the size; units/fanin override it only when set to | |
| # a positive number. Passing raw numbers unconditionally would make | |
| # every run's size an accident of this file rather than a named, | |
| # comparable workload — and --preset must come first so the | |
| # overrides still win. | |
| args+=( --preset "${{ inputs.preset || matrix.preset }}" ) | |
| [ "${{ inputs.units || 0 }}" -gt 0 ] 2>/dev/null && args+=( --units "${{ inputs.units }}" ) | |
| [ "${{ inputs.fanin || 0 }}" -gt 0 ] 2>/dev/null && args+=( --fanin "${{ inputs.fanin }}" ) | |
| else | |
| # A real tree: measured in place, and the scenarios that perturb a | |
| # file must be TOLD which one. Without --hub/--leaf/--body they | |
| # report `skipped` with the reason rather than picking a file and | |
| # producing a number that looks valid. | |
| args+=( --project "$BENCH_PROJECT" | |
| --buildfiles "$GITHUB_WORKSPACE/bench/projects/${{ matrix.project }}" ) | |
| case "${{ matrix.project }}" in | |
| mcpp) args+=( --hub "src/platform/platform.cppm" | |
| --body "src/version_req.cppm" ) ;; | |
| xlings) args+=( --hub "src/xlings.cppm" | |
| --body "src/xlings.cppm" ) ;; | |
| esac | |
| fi | |
| "$BENCH" "${args[@]}" | |
| - name: Upload report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.project }} | |
| path: bench-${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.project }}.json | |
| if-no-files-found: error |