diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e96666b4..f12cc10e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -12,8 +12,36 @@ on: # nightly full regression — exercises every workspace member regardless of diff - cron: "0 6 * * *" workflow_dispatch: + inputs: + cache: + description: "Package build cache — 'local' rebuilds every dependency per member, which is what the timing table should be read against when comparing" + type: choice + options: [global, local] + default: global env: + # 2026.8.5.4 carries two things this workflow depends on: + # .5.1 `tools = [...]` — how a consumer asks for a dependency's + # `kind = "bin"` target, which tests/examples/protobuf-protoc is built + # on. Before it: "tools must be a string, inline dep table, or nested + # table". + # .5.4 windows links with lld. link.exe caps a response-file LINE at + # 128 KiB and opencv-module / opencv-module-dnn went past it — + # fatal error LNK1170: line in command file contains 135135 or + # more characters + # after 795s / 1166s of compiling. .5.3 newline-separated OUR response + # file, which was necessary but not sufficient: clang, acting as the + # driver, writes a SECOND one for the linker that we do not control. + # lld's response-file parser has no per-line limit at all. + # Together with re-enabling the global package cache below, this is + # what makes a green FULL run possible again: .5.3 removes the + # windows link failure, the cache removes the 150-minute timeout. + # + # Neither of them moves index.toml's min_mcpp: exposing compat.protobuf's `protoc` + # target is additive, and 2026.8.3.3 still parses that descriptor with an + # empty unknown_keys. The floor an index publishes decides whether older + # clients keep working at all (mcpp#349), so it moves only when a descriptor + # genuinely stops being readable — which is not the case here. # 2026.8.3.1: on macOS, a global object that touches std::cout during static # init crashes on sight (mcpp#336). Mach-O has no priority-ordered init # section and libc++'s carries no ios_base::Init guard of its own, @@ -88,7 +116,7 @@ env: # 0.0.94 fixed feature-gated `sources` under `mcpp test` (mcpp#218); 0.0.91 # added standard = "c++fly" to the resolver grammar, so c++fly descriptors # get the lint WARN below, not a hard grammar-parse rejection. - MCPP_VERSION: "2026.8.3.3" + MCPP_VERSION: "2026.8.5.4" jobs: lint: @@ -236,110 +264,23 @@ jobs: # `dnn` feature members. The registry cache (restore-keys prefix below) # amortizes those across subsequent runs. 150 covers the one-time cold full # build with headroom; it is a ceiling, not a target. - workspace: - name: workspace (${{ matrix.platform }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 150 - strategy: - fail-fast: false - matrix: - include: - # Archive names are derived from env.MCPP_VERSION in the Download - # step — bumping the pin is a ONE-line change (hardcoded versions - # here once 404'd a pin bump). - - platform: linux - os: ubuntu-latest - suffix: linux-x86_64 - ext: tar.gz - mcpp: bin/mcpp - xlings: registry/bin/xlings - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION - - platform: macos - os: macos-15 - suffix: macosx-arm64 - ext: tar.gz - mcpp: bin/mcpp - xlings: registry/bin/xlings - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION - - platform: windows - os: windows-latest - suffix: windows-x86_64 - ext: zip - mcpp: bin/mcpp.exe - xlings: registry/bin/xlings.exe - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION - env: - MCPP_EFFECTIVE: ${{ matrix.mcpp_version }} + # ── The plan, computed ONCE ─────────────────────────────────────────── + # Was inlined in every workspace job — three runners each re-deriving the + # same answer. It now also has to be decided BEFORE the matrix exists, + # because the matrix's shard dimension depends on it: a full run fans out, + # a selective one does not. + select: + runs-on: ubuntu-latest + outputs: + members: ${{ steps.fanout.outputs.members }} + matrix: ${{ steps.fanout.outputs.matrix }} + plan: ${{ steps.plan_shards.outputs.plan }} steps: - # Full history: the member-selection step below diffs against the PR - # base to decide which workspace members to test. - uses: actions/checkout@v4 with: fetch-depth: 0 - # The cache key is computed ONCE, here, instead of inline in the cache - # step. `hashFiles()` globs the WORKING TREE, and actions/cache - # re-evaluates its `key` in the post (save) step — i.e. AFTER the build, - # when `tests/**` no longer matches 80-odd tracked sources but tens of - # thousands of build-output files under tests/examples/*/target and - # .mcpp (multi-GB; .gitignore does not apply to hashFiles). Hashing that - # tree blew past the runner's 120s template-evaluation cap on windows - # and failed an otherwise all-green job: - # "hashFiles('pkgs/**/*.lua, tests/**, .github/workflows/validate.yml') - # couldn't finish within 120 seconds" - # `git ls-files -s` reads the INDEX, so it sees exactly the tracked - # inputs, never build output, and reports blob SHAs git already has — - # no file content is read at all. Freezing the result in the job env - # also guarantees the save step keys on the same string the restore - # step used, no matter what the build left behind. - - name: Compute registry cache key - shell: bash - run: | - # git hash-object rather than sha256sum/cut: git is already a hard - # requirement here (checkout ran), coreutils on the windows leg is - # only a Git-Bash convenience. - h=$(git ls-files -s -- 'pkgs/**/*.lua' 'tests/**' '.github/workflows/validate.yml' \ - | git hash-object --stdin) - echo "REGISTRY_CACHE_KEY=mcpp-registry-${{ runner.os }}-${{ env.MCPP_EFFECTIVE }}-$h" >> "$GITHUB_ENV" - - name: Restore mcpp registry cache - uses: actions/cache@v4 - with: - # Holds toolchains AND the built compat packages (data/xpkgs), so a - # repeat `mcpp test` rebuilds little. - path: ~/.mcpp/registry - key: ${{ env.REGISTRY_CACHE_KEY }} - restore-keys: | - mcpp-registry-${{ runner.os }}-${{ env.MCPP_EFFECTIVE }}- - - name: Download mcpp - shell: bash - env: - MCPP_ARCHIVE: mcpp-${{ env.MCPP_EFFECTIVE }}-${{ matrix.suffix }}.${{ matrix.ext }} - MCPP_ROOT: mcpp-${{ env.MCPP_EFFECTIVE }}-${{ matrix.suffix }} - run: | - curl -L -fsS -o "$MCPP_ARCHIVE" \ - "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_EFFECTIVE}/${MCPP_ARCHIVE}" - case "$MCPP_ARCHIVE" in - *.zip) powershell -NoProfile -Command "Expand-Archive -Force -Path '${MCPP_ARCHIVE}' -DestinationPath '.'" ;; - *) tar -xzf "$MCPP_ARCHIVE" ;; - esac - root="$PWD/$MCPP_ROOT" - mkdir -p "$HOME/.mcpp/registry" - cp -a "$root/registry/." "$HOME/.mcpp/registry/" - if [[ "$RUNNER_OS" == "Windows" ]]; then - echo "MCPP=$(cygpath -m "$root/${{ matrix.mcpp }}")" >> "$GITHUB_ENV" - echo "MCPP_VENDORED_XLINGS=$(cygpath -m "$root/${{ matrix.xlings }}")" >> "$GITHUB_ENV" - echo "$(cygpath -m "$root/bin")" >> "$GITHUB_PATH" - else - echo "MCPP=$root/${{ matrix.mcpp }}" >> "$GITHUB_ENV" - echo "MCPP_VENDORED_XLINGS=$root/${{ matrix.xlings }}" >> "$GITHUB_ENV" - echo "$root/bin" >> "$GITHUB_PATH" - fi - # compat.ffmpeg / compat.opencv5 carry NASM .asm sources. No host - # install and no index-refresh pre-step needed: mcpp >= 0.0.97 - # resolves nasm itself through the same synchronous gate as the - # toolchain (index refresh + install + payload check BEFORE the build - # plans, mcpp#232). The sandbox copy lands in ~/.mcpp/registry, so - # the cache carries it across runs. - + - name: Install lua + run: sudo apt-get install -y --no-install-recommends lua5.4 # ── Selective member testing ────────────────────────────────────── # `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and # dominates CI wall-clock, while a PR almost always touches one @@ -354,6 +295,7 @@ jobs: # changes select nothing. # Note: bash 3.2 on macOS runners — no associative arrays here. - name: Select affected workspace members + id: plan shell: bash run: | full() { echo "MEMBERS=__ALL__" >> "$GITHUB_ENV"; echo "full run: $1"; exit 0; } @@ -408,6 +350,179 @@ jobs: sel=${sel# } echo "MEMBERS=$sel" >> "$GITHUB_ENV" echo "selected members: ${sel:-}" + # Sharding is for the FULL run only, and the shard count per platform is + # that platform's RUNNER CONCURRENCY — not a round number. + # + # Measured on this repo (24 jobs queued, 6 running): + # macos 1 · linux 3 · windows 2 + # + # That measurement is what makes over-sharding a real cost rather than a + # theoretical one: at concurrency 1, eight macOS shards run BACK TO BACK + # and each pays its own checkout + mcpp download + cache restore, so the + # split is strictly slower than not splitting. Wall-clock is + # ceil(shards / concurrency) x slowest-shard; shards beyond the + # concurrency only add fixed cost. + # + # Re-measure with: + # gh api repos///actions/runs//jobs --paginate \ + # --jq '[.jobs[]|select(.status=="in_progress")]|length' + - name: Decide the fan-out + id: fanout + shell: bash + run: | + full=0; [ "$MEMBERS" = "__ALL__" ] && full=1 + emit() { # platform os suffix ext mcpp xlings shards + for i in $(seq 0 $(( $7 - 1 ))); do + printf '{"platform":"%s","os":"%s","suffix":"%s","ext":"%s","mcpp":"%s","xlings":"%s","shard":%d,"shards":%d},' \ + "$1" "$2" "$3" "$4" "$5" "$6" "$i" "$7" + done + } + if [ "$full" = 1 ]; then ln=3; mn=1; wn=2; else ln=1; mn=1; wn=1; fi + { + printf '{"include":[' + emit linux ubuntu-latest linux-x86_64 tar.gz bin/mcpp registry/bin/xlings "$ln" + emit macos macos-15 macosx-arm64 tar.gz bin/mcpp registry/bin/xlings "$mn" + emit windows windows-latest windows-x86_64 zip bin/mcpp.exe registry/bin/xlings.exe "$wn" + printf ']}' + } | sed 's/,]}/]}/' > /tmp/matrix.json + echo "matrix=$(cat /tmp/matrix.json)" >> "$GITHUB_OUTPUT" + echo "members=$MEMBERS" >> "$GITHUB_OUTPUT" + cat /tmp/matrix.json + + # The split is computed ONCE, here, and shipped to the runners as data. + # It used to run on each runner, which needed lua5.4 on all three + # platforms — windows has no apt or brew, and macOS's brew installs + # `lua`, not `lua5.4`, so every non-linux shard died with + # `lua5.4: command not found` after 16 seconds. Deciding once is also + # simply correct: one plan, not three runners each re-deriving it. + - name: Plan the shards + id: plan_shards + shell: bash + run: | + plan='${{ steps.fanout.outputs.members }}' + [ "$plan" = "__ALL__" ] && plan="" + { + printf '{' + first=1 + for spec in linux:$(jq -r '[.include[]|select(.platform=="linux")]|length' /tmp/matrix.json) \ + macos:$(jq -r '[.include[]|select(.platform=="macos")]|length' /tmp/matrix.json) \ + windows:$(jq -r '[.include[]|select(.platform=="windows")]|length' /tmp/matrix.json); do + p=${spec%%:*}; n=${spec##*:} + [ "$first" = 1 ] || printf ',' + first=0 + printf '"%s":{' "$p" + for i in $(seq 0 $((n - 1))); do + [ "$i" = 0 ] || printf ',' + m=$(lua5.4 tests/plan_shards.lua "$p" "$i" "$n" $plan) + printf '"%s":"%s"' "$i" "$m" + done + printf '}' + done + printf '}' + } > /tmp/plan.json + echo "plan=$(cat /tmp/plan.json)" >> "$GITHUB_OUTPUT" + jq . /tmp/plan.json + + workspace: + # The shard suffix appears only when the platform is actually split. + name: workspace (${{ matrix.platform }}${{ matrix.shards == 1 && '' || format(' {0}/{1}', matrix.shard, matrix.shards) }}) + needs: select + if: needs.select.outputs.members != '' + runs-on: ${{ matrix.os }} + # One shard is a fraction of the work, so this is a real ceiling rather + # than the thing that decides whether the job finishes (a full linux run + # used to hit 150 exactly and get cancelled). + timeout-minutes: 90 + strategy: + fail-fast: false + # Whole matrix from `select`: the shard count is per-platform, because it + # tracks that platform's runner concurrency. + matrix: ${{ fromJSON(needs.select.outputs.matrix) }} + steps: + # Full history: the member-selection step below diffs against the PR + # base to decide which workspace members to test. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # The cache key is computed ONCE, here, instead of inline in the cache + # step. `hashFiles()` globs the WORKING TREE, and actions/cache + # re-evaluates its `key` in the post (save) step — i.e. AFTER the build, + # when `tests/**` no longer matches 80-odd tracked sources but tens of + # thousands of build-output files under tests/examples/*/target and + # .mcpp (multi-GB; .gitignore does not apply to hashFiles). Hashing that + # tree blew past the runner's 120s template-evaluation cap on windows + # and failed an otherwise all-green job: + # "hashFiles('pkgs/**/*.lua, tests/**, .github/workflows/validate.yml') + # couldn't finish within 120 seconds" + # `git ls-files -s` reads the INDEX, so it sees exactly the tracked + # inputs, never build output, and reports blob SHAs git already has — + # no file content is read at all. Freezing the result in the job env + # also guarantees the save step keys on the same string the restore + # step used, no matter what the build left behind. + - name: Compute registry cache key + shell: bash + run: | + # git hash-object rather than sha256sum/cut: git is already a hard + # requirement here (checkout ran), coreutils on the windows leg is + # only a Git-Bash convenience. + h=$(git ls-files -s -- 'pkgs/**/*.lua' 'tests/**' '.github/workflows/validate.yml' \ + | git hash-object --stdin) + echo "REGISTRY_CACHE_KEY=mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}-$h" >> "$GITHUB_ENV" + - name: Restore mcpp registry cache + uses: actions/cache@v4 + with: + # Holds toolchains AND the built compat packages (data/xpkgs), so a + # repeat `mcpp test` rebuilds little. + path: ~/.mcpp/registry + key: ${{ env.REGISTRY_CACHE_KEY }} + restore-keys: | + mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}- + - name: Download mcpp + shell: bash + env: + MCPP_ARCHIVE: mcpp-${{ env.MCPP_VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }} + MCPP_ROOT: mcpp-${{ env.MCPP_VERSION }}-${{ matrix.suffix }} + run: | + curl -L -fsS -o "$MCPP_ARCHIVE" \ + "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_VERSION}/${MCPP_ARCHIVE}" + case "$MCPP_ARCHIVE" in + *.zip) powershell -NoProfile -Command "Expand-Archive -Force -Path '${MCPP_ARCHIVE}' -DestinationPath '.'" ;; + *) tar -xzf "$MCPP_ARCHIVE" ;; + esac + root="$PWD/$MCPP_ROOT" + mkdir -p "$HOME/.mcpp/registry" + cp -a "$root/registry/." "$HOME/.mcpp/registry/" + if [[ "$RUNNER_OS" == "Windows" ]]; then + echo "MCPP=$(cygpath -m "$root/${{ matrix.mcpp }}")" >> "$GITHUB_ENV" + echo "MCPP_VENDORED_XLINGS=$(cygpath -m "$root/${{ matrix.xlings }}")" >> "$GITHUB_ENV" + echo "$(cygpath -m "$root/bin")" >> "$GITHUB_PATH" + else + echo "MCPP=$root/${{ matrix.mcpp }}" >> "$GITHUB_ENV" + echo "MCPP_VENDORED_XLINGS=$root/${{ matrix.xlings }}" >> "$GITHUB_ENV" + echo "$root/bin" >> "$GITHUB_PATH" + fi + # compat.ffmpeg / compat.opencv5 carry NASM .asm sources. No host + # install and no index-refresh pre-step needed: mcpp >= 0.0.97 + # resolves nasm itself through the same synchronous gate as the + # toolchain (index refresh + install + payload check BEFORE the build + # plans, mcpp#232). The sandbox copy lands in ~/.mcpp/registry, so + # the cache carries it across runs. + + # ── This shard's slice of the plan ──────────────────────────────── + # `select` decided WHAT runs; this decides which part of it runs HERE. + # Round-robin by position, which is what spreads the expensive members: + # opencv-module / -dnn / -unifont are adjacent in the list, so `% N` + # necessarily puts them on three different runners. A single job that + # builds all three spends 45+ minutes on opencv alone. + # ── This shard's slice ──────────────────────────────────────────── + # Already decided by `select` (measured-time bin packing, see + # tests/plan_shards.lua). Arrives as data, so a runner needs no lua. + - name: Take this shard's members + shell: bash + run: | + mine='${{ fromJSON(needs.select.outputs.plan)[matrix.platform][format('{0}', matrix.shard)] }}' + echo "MEMBERS=$mine" >> "$GITHUB_ENV" + echo "shard ${{ matrix.shard }}/${{ matrix.shards }}: ${mine:-}" # ── Refresh the PUBLISHED index before testing ──────────────────── # Most members resolve everything from this checkout, but a member that @@ -437,38 +552,58 @@ jobs: shell: bash env: MCPP_INDEX_MIRROR: GLOBAL - # Dependencies build inside each member's own target/ instead of - # through the global package build cache (mcpp >= 2026.7.30.2). - # That cache is unusable here: mcpp#233's object-path disambiguation - # fires on basename collisions across the WHOLE build dir — i.e. on - # which packages the CONSUMER pulls in — while the cache key covers - # only the dependency itself, so one entry can hold two different - # layouts. `tests/examples/archive` pulls zlib AND bzip2 (both ship - # compress.c) and stores obj/compat_zlib/zlib-1.3.2/compress.o; - # every zlib consumer without bzip2 then asks the same key for a - # flat obj/compress.o and ninja dies at graph time with - # "missing and no known rule to make it". Reproduced both ways round - # on 2026.8.3.3 and filed as mcpp-community/mcpp#344; drop this once - # it lands. `local` still caches the std BMI, which is the expensive - # one — only package entries are bypassed. - MCPP_BUILD_CACHE: local + # The GLOBAL package build cache is on (mcpp >= 2026.7.30.2), which + # is the default — this step used to set `MCPP_BUILD_CACHE: local` + # and no longer does. + # + # That bypass existed for mcpp#344: object-path disambiguation fires + # on basename collisions across the WHOLE build dir — i.e. on what + # the CONSUMER pulls in — while the cache key covered only the + # dependency, so one entry could hold two layouts and ninja died at + # graph time with "missing and no known rule to make it". #344 + # landed in 2026.8.3.4 with per-package Merkle keys that cover the + # consumer-dependent layout, so the reason is gone. + # + # Keeping it cost real time, and the full run is where it showed: + # with `local`, EVERY member recompiles EVERY dependency from + # scratch. 59 members that mostly share abseil / protobuf / opencv + # meant the same sources were built over and over — + # + # linux 2h30m -> cancelled at the 150-minute timeout + # windows 2h20m + # macos 1h26m + # + # — and a workspace cannot be validated by a job that cannot finish. + # With the cache on, a given (package, version, features, toolchain) + # is built once per run and every later member hits it. run: | "$MCPP" --version # No `timeout` wrapper: absent on macOS runners; job-level timeout-minutes bounds it. - if [ "$MEMBERS" = "__ALL__" ]; then - "$MCPP" test --workspace - elif [ -z "$MEMBERS" ]; then + # One code path: the shard step above already expanded `__ALL__` + # into this runner's actual member names, so `mcpp test --workspace` + # — which would ignore the sharding and rebuild everything here — is + # gone. + # + # tests/run_members.sh is the SAME script you run locally. A timing + # table that only exists in CI cannot be used while deciding what to + # optimise, and a local harness that differs from CI measures + # something else. + if [ -z "$MEMBERS" ]; then echo "No workspace member affected by this change — nothing to test." else - rc=0 - for m in $MEMBERS; do - echo "::group::mcpp test -p $m" - "$MCPP" test -p "$m" || rc=1 - echo "::endgroup::" - done - exit $rc + MCPP_TIMINGS="$PWD/timings.tsv" bash tests/run_members.sh $MEMBERS fi + # Per-shard timings, merged by the `timings` job below. `always()`: a + # run that failed is exactly when knowing where the time went matters. + - name: Upload this shard's timings + if: always() && hashFiles('timings.tsv') != '' + uses: actions/upload-artifact@v4 + with: + name: timings-${{ matrix.platform }}-${{ matrix.shard }} + path: timings.tsv + retention-days: 14 + # install()-driven packages (openssl, openblas) build through their own # Make/Configure system, whose output xim's interface mode swallows; a # failed hook surfaces only as `E_INTERNAL: [] failed:`. Each writes @@ -487,3 +622,83 @@ jobs: echo "::endgroup::" done < <(find tests/examples "$HOME/.mcpp/registry" -name 'mcpp_*_build.log' 2>/dev/null) [ "$found" = 1 ] || echo "no install() build logs found" + + # ── Where the time went ─────────────────────────────────────────────── + # Sharding hides the cost: eight runners each report their own slice, and + # nobody can see which members actually dominate. This merges them into one + # ranking per platform, in the run summary, so the next optimisation starts + # from measurement instead of a guess. + # + # `always()` — a failed run is exactly when this is worth reading. + timings: + needs: [select, workspace] + if: always() && needs.select.outputs.members != '' + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + pattern: timings-* + path: timings + continue-on-error: true + - name: Rank members by wall-clock + shell: bash + run: | + shopt -s nullglob + files=(timings/*/timings.tsv) + if [ ${#files[@]} -eq 0 ]; then + echo "no timing data (every shard skipped or failed before testing)" \ + >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Artifact name carries the platform: timings--. + for plat in linux macos windows; do + rows=$(mktemp) + for f in timings/timings-$plat-*/timings.tsv; do + [ -f "$f" ] && cat "$f" >> "$rows" + done + [ -s "$rows" ] || { rm -f "$rows"; continue; } + + total=$(awk -F'\t' '{s += $1} END {print s+0}' "$rows") + count=$(wc -l < "$rows") + { + echo "### $plat — ${count} member(s), ${total}s of member wall-clock" + echo + echo "| rank | seconds | share | member | result |" + echo "|---:|---:|---:|---|---|" + sort -rn "$rows" | awk -F'\t' -v tot="$total" ' + { pct = tot > 0 ? ($1 * 100 / tot) : 0 + printf "| %d | %s | %.1f%% | `%s` | %s |\n", NR, $1, pct, $2, $3 }' + echo + } >> "$GITHUB_STEP_SUMMARY" + rm -f "$rows" + done + + echo "_Total is the SUM across shards; wall-clock is the slowest shard._" \ + >> "$GITHUB_STEP_SUMMARY" + + # The table that feeds the NEXT run's sharding. Emitted as an + # artifact rather than committed automatically: a number that + # rewrites itself on every run would make every diff noisy and would + # silently absorb a one-off slow runner. Refresh it deliberately — + # download this artifact and replace tests/member-timings.tsv when + # the numbers have actually moved. + { + echo "# \t\t — from run ${{ github.run_id }}" + echo "# refresh: download the member-timings artifact and replace this file" + for plat in linux macos windows; do + for f in timings/timings-$plat-*/timings.tsv; do + [ -f "$f" ] || continue + awk -F'\t' -v p="$plat" '{ printf "%s\t%s\t%s\n", p, $2, $1 }' "$f" + done + done + } | sort -u > member-timings.tsv + echo "wrote member-timings.tsv ($(grep -vc '^#' member-timings.tsv) rows)" + + - name: Upload the timing table for the next run's sharding + if: always() && hashFiles('member-timings.tsv') != '' + uses: actions/upload-artifact@v4 + with: + name: member-timings + path: member-timings.tsv + retention-days: 90 diff --git a/README.md b/README.md index 2ecb6b3d..429508ae 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Two kinds of packages live here: |------|------| | Native module library (Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](pkgs/o/opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) | | C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | +| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | | C++-source compat, zero-dep client + optional components | [`compat.websocket`](pkgs/c/compat.websocket.lua) (IXWebSocket 12.0.1 — a pure RFC 6455 client compiled from upstream's `IXWEBSOCKET_SOURCES` minus the four server TUs, so the **base build has zero external dependencies**: TLS off (the OpenSSL/MbedTLS/AppleSSL TUs aren't built) and `IXWEBSOCKET_USE_ZLIB` unset, so the gzip codec compiles to a no-op. Two optional features add on top: `server` (the four server TUs — `IXWebSocketServer`, `IXSocketServer`, `IXHttpServer`, `IXWebSocketProxyServer` — needing nothing external, and it **implies `zlib`** because upstream's server advertises permessage-deflate by default, which the transport negotiates regardless of the define) and `zlib` (deps `compat.zlib` and turns the codec into real per-message-deflate compression). The default-feature test brings its own minimal RFC 6455 echo server on loopback sockets (handshake, masking, fragmentation and close all exercised offline); a second member, `websocket-features`, runs a real `ix::WebSocketServer` and asserts the compression is observable on the wire — a 64 KiB repeated payload round-trips with `wireSize` = 80) | | header-only (with `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | Runtime loader compat (pure sources, sidestepping upstream codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua) (the Khronos loader: `loader/generated/` is checked in, and the assembly path degrades to plain C through `UNKNOWN_FUNCTIONS_SUPPORTED`, so no CMake/Python/assembler is needed; windows deferred) · [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) | diff --git a/README.zh-CN.md b/README.zh-CN.md index 1b107812..97cfe12d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -36,7 +36,7 @@ mcpp self config --mirror CN # 切换至国内镜像,默认使用 GLOBAL 上 |------|------| | 原生模块库(Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](pkgs/o/opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)| | C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | +| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | | C++ 源码 compat(零依赖客户端 + 可选组件) | [`compat.websocket`](pkgs/c/compat.websocket.lua)(IXWebSocket 12.0.1 —— 从上游 `IXWEBSOCKET_SOURCES` 剔掉 4 个 server TU 后直编的纯 RFC 6455 客户端,**基座零外部依赖**:TLS 关闭(OpenSSL/MbedTLS/AppleSSL 三组 TU 均不编),`IXWEBSOCKET_USE_ZLIB` 不定义(gzip codec 编译为 no-op)。两个可选 feature 在基座上叠加:`server`(4 个 server TU —— `IXWebSocketServer`/`IXSocketServer`/`IXHttpServer`/`IXWebSocketProxyServer`,零新增外部依赖,且 **implies `zlib`** —— 因为上游 server 默认就宣称 permessage-deflate,而 transport 的协商不受宏门控)与 `zlib`(依赖 `compat.zlib`,把 codec 变成真正的 permessage-deflate 压缩)。默认构建的测试自带基于 loopback 原始 socket 的最小 RFC 6455 echo server(握手/掩码/分片/关闭全部离线实测);第二个成员 `websocket-features` 跑真实的 `ix::WebSocketServer`,并断言压缩在线路上可观测 —— 64 KiB 重复载荷往返,`wireSize` = 80) | | header-only(含 `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | 运行时 loader compat(纯源码,绕开上游 codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua)(Khronos loader:`loader/generated/` 已签入,汇编路径经 `UNKNOWN_FUNCTIONS_SUPPORTED` 降级为纯 C,故无需 CMake/Python/汇编器;windows 延后)· [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) | diff --git a/docs/package-types.md b/docs/package-types.md index d348f493..3930a093 100644 --- a/docs/package-types.md +++ b/docs/package-types.md @@ -18,6 +18,7 @@ combined as needed. | **E. Whole-source direct build with a generated config** | upstream generates its config header through configure/CMake; here a snapshot of it lands in `generated_files` | `pkgs/c/compat.libpng.lua`, `compat.curl.lua`, `compat.sdl2.lua`, `compat.ffmpeg.lua` | `generated_files` + `include_dirs` | | **F. Shared-library compat** | has to be the **only** copy of that `.so` in the process (third parties `dlopen` it) | the X11 family such as `pkgs/c/compat.x11.lua`, and `compat.vulkan.lua` (linux) | `targets = { kind = "shared", soname = … }` | | **G. Host runtime adaptation** | things that cannot be vendored, such as drivers — only a symlink farm plus metadata | `pkgs/c/compat.glx-runtime.lua`, `compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` | +| **H. Host tool provider** | the upstream tarball also holds a **code generator** consumers run at build time | `pkgs/c/compat.protobuf.lua` (`protoc`) | a `targets` entry with `kind = "bin"` + `main`, plus `required_features` | For the complete sample index, see the [Reference examples table in the root README](../README.md#reference-examples-lua-descriptors). @@ -200,6 +201,50 @@ Two details that keep biting: - **The closure has to be complete.** A farm holding `libxcb.so.1` but not the `libXau.so.6` it depends on shadows the host copy that would otherwise have resolved, and the executable simply fails to start. +## H. Host tool provider (`compat.protobuf`'s `protoc`) + +Some tarballs hold both a library and the code generator that emits code against it. Declare the generator as a second +target, and consumers ask for it with `tools = [...]` (mcpp 2026.8.5.1+): + +```lua +targets = { + ["protobuf"] = { kind = "lib" }, + ["protoc"] = { kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" } }, +}, +features = { + ["protoc"] = { sources = { … the compiler's own sources … } }, +}, +``` + +```toml +# consumer side — one dependency, two roles +compat.protobuf = { version = "35.1", tools = ["protoc"] } +``` + +mcpp then builds that target **for the build machine** in a nested sub-build and hands the path to the consumer's +`build.mcpp` through `mcpp::dep_bin("protobuf", "protoc")`. This is the whole reason the shape is worth naming: the +tool's version **is** the dependency's version, so a generator/runtime mismatch — a *runtime* failure everywhere else, +and the classic protobuf footgun — is not expressible. Under `mcpp build --target ` the tool is still built for +the host, because a code generator has to run here. + +Four things to get right: + +- **Gate the compiler's sources behind a feature**, and name it in the target's `required_features`. Consumers who only + link the library must not pay for the generator's TUs; consumers who ask for the tool must not have to know which + features it needs. `compat.protobuf`'s `protoc` also requires `upb`, because libprotoc's upb generator links the upb + runtime — get that wrong and it fails at **link** time with missing `upb_*` symbols. +- **Transcribe the source list from upstream**, exactly as for a library — protobuf's 138 entries come from + `libprotoc_srcs` in its own `src/file_lists.cmake`. +- **`main` needs the same `*/` wrap glob as `sources`**; it is expanded the same way. +- **A generator that reads data files at runtime still needs a path to them.** protoc does not embed the well-known + types: `import "google/protobuf/timestamp.proto"` is read from disk. Consumers derive that directory from + `mcpp::dep_dir("protobuf")` — see `tests/examples/protobuf-protoc/build.mcpp`. + +The matching member is `tests/examples/protobuf-protoc`, and it is the complement of `tests/examples/protobuf`: that +one deliberately uses no generated code, this one is generated code end to end. + --- ## The minimal project (`tests/examples//`) diff --git a/docs/zh/package-types.md b/docs/zh/package-types.md index 4ffc00eb..becdcd33 100644 --- a/docs/zh/package-types.md +++ b/docs/zh/package-types.md @@ -16,6 +16,7 @@ A–D 是四种**基础**形态,先按它们判定;E–G 是在基础形态之 | **E. 生成 config 的全源码直编** | 上游用 configure/CMake 生成配置头,此处以 `generated_files` 落一份快照 | `pkgs/c/compat.libpng.lua`、`compat.curl.lua`、`compat.sdl2.lua`、`compat.ffmpeg.lua` | `generated_files` + `include_dirs` | | **F. 共享库 compat** | 必须是**唯一**的那个 `.so`(会被第三方 `dlopen`) | `pkgs/c/compat.x11.lua` 等 X11 家族、`compat.vulkan.lua`(linux) | `targets = { kind = "shared", soname = … }` | | **G. 宿主运行时适配** | 驱动之类无法 vendor 的东西,只做符号链接农场 + 元数据 | `pkgs/c/compat.glx-runtime.lua`、`compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` | +| **H. 宿主工具提供方** | 上游 tarball 里除了库,还带着消费者在构建期要跑的**代码生成器** | `pkgs/c/compat.protobuf.lua`(`protoc`) | `targets` 里一条 `kind = "bin"` + `main`,配 `required_features` | 完整的样例索引见[根 README 的「参考示例」表](../../README.zh-CN.md#参考示例lua-描述符)。 @@ -184,6 +185,48 @@ runtime = { - **闭包必须完整**。农场里有 `libxcb.so.1` 却没有它依赖的 `libXau.so.6`,会遮蔽掉本来能解析的宿主副本,可执行 文件直接起不来。 +## H. 宿主工具提供方(`compat.protobuf` 的 `protoc`) + +有些 tarball 里同时装着一个库,和「针对这个库生成代码」的那个生成器。把生成器声明成第二个 target, +消费者用 `tools = [...]` 索取(mcpp 2026.8.5.1 起): + +```lua +targets = { + ["protobuf"] = { kind = "lib" }, + ["protoc"] = { kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" } }, +}, +features = { + ["protoc"] = { sources = { … 编译器自身的源码 … } }, +}, +``` + +```toml +# 消费者侧 —— 一条依赖,两种角色 +compat.protobuf = { version = "35.1", tools = ["protoc"] } +``` + +mcpp 会在一次嵌套子构建里把这个 target 编成**构建机**的二进制,并把路径经 +`mcpp::dep_bin("protobuf", "protoc")` 交给消费者的 `build.mcpp`。这个形态值得单列的全部理由在于: +工具的版本**就是**那条依赖的版本,于是「生成器与运行时版本错配」——在别处是**运行期**才炸、也正是 +protobuf 最经典的坑——在这里**语法上无法表达**。`mcpp build --target ` 下工具仍为宿主构建, +因为代码生成器必须在本机跑。 + +四个要点: + +- **把编译器的源码关进一个 feature**,并在 target 的 `required_features` 里写明。只链库的消费者不该为 + 生成器的 TU 买单;索取工具的消费者也不该需要知道它要哪些 feature。`compat.protobuf` 的 `protoc` 还 + 必须要 `upb`,因为 libprotoc 的 upb 生成器要链 upb 运行时——搞错了会在**链接期**缺一批 `upb_*` 符号。 +- **源码列表照旧逐条转录自上游**:protobuf 这 138 项来自它自己的 `src/file_lists.cmake` 的 `libprotoc_srcs`。 +- **`main` 和 `sources` 一样需要 `*/` 那层 wrap glob**,展开方式相同。 +- **运行期还要读数据文件的生成器,仍然需要一个路径**。protoc 并不内嵌 well-known types: + `import "google/protobuf/timestamp.proto"` 是从磁盘读的。消费者用 `mcpp::dep_dir("protobuf")` 推出那个 + 目录——见 `tests/examples/protobuf-protoc/build.mcpp`。 + +对应的成员是 `tests/examples/protobuf-protoc`,它与 `tests/examples/protobuf` 互为补集:那个刻意**不用** +任何生成代码,这个从头到尾都是生成代码。 + --- ## 最小工程(`tests/examples//`) diff --git a/mcpp.toml b/mcpp.toml index 908d98da..f192b49f 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -47,6 +47,7 @@ members = [ "tests/examples/protobuf", "tests/examples/protobuf-upb", "tests/examples/protobuf-gzip", + "tests/examples/protobuf-protoc", "tests/examples/opencv-module", "tests/examples/opencv-module-dnn", "tests/examples/opencv-module-unifont", diff --git a/pkgs/c/compat.protobuf.lua b/pkgs/c/compat.protobuf.lua index fb5ab549..7d7c7b01 100644 --- a/pkgs/c/compat.protobuf.lua +++ b/pkgs/c/compat.protobuf.lua @@ -14,14 +14,22 @@ -- source list covers linux/macosx/windows and the three xpm blocks share -- a single tarball and sha256. -- --- SCOPE — runtime only. This package builds upstream's `libprotobuf` target --- (79 TUs), i.e. what a program that *uses* generated code needs: messages, --- reflection, descriptors, text/JSON formats, the well-known types. It does --- NOT build `libprotoc` (a further 157 TUs) and ships no protoc binary, so it --- does not generate .pb.cc from .proto. Consumers either check in --- protoc-generated sources or build them with the official upstream protoc --- release (protoc-35.1-.zip); wiring that into an mcpp build belongs --- to a build.mcpp step, not to this descriptor. +-- SCOPE — runtime by default, compiler on request. This package builds +-- upstream's `libprotobuf` target (79 TUs) unconditionally: messages, +-- reflection, descriptors, text/JSON formats, the well-known types — what a +-- program that *uses* generated code needs. +-- +-- Since mcpp 2026.8.5.1 it ALSO offers `protoc` as a host tool, behind the +-- `protoc` feature (upstream's `libprotoc`, 138 further TUs). A consumer that +-- only links the runtime compiles none of them: +-- +-- compat.protobuf = { version = "35.1", tools = ["protoc"] } +-- +-- That replaces the old advice of "check in protoc output, or fetch the +-- official protoc-35.1-.zip and keep its version in step by hand". +-- Keeping it in step by hand is precisely the failure this removes: a protoc +-- that disagrees with the runtime fails at RUNTIME, and here the tool's +-- version IS this package's version, so the mismatch cannot be expressed. -- -- Version numbering follows upstream verbatim: `35.1` is the protobuf release, -- and it is what gRPC 1.83.0 pins (its third_party/protobuf submodule is @@ -194,13 +202,187 @@ package = { "*/third_party/utf8_range/utf8_range.c", }, - targets = { ["protobuf"] = { kind = "lib" } }, + targets = { + ["protobuf"] = { kind = "lib" }, + -- #355 (mcpp 2026.8.5.1+): protoc as a HOST tool a consumer can ask + -- for, so it never has to supply a matching one by hand: + -- + -- compat.protobuf = { version = "35.1", tools = ["protoc"] } + -- + -- The version axis is what matters here. protoc generating code for + -- a DIFFERENT protobuf runtime than the one being linked fails at + -- RUNTIME, not at compile time, and is the single nastiest thing + -- about hand-managed protobuf codegen. Because the tool's version + -- IS this package's version, that mismatch is not expressible. + -- + -- `required_features` is a GATE in an ordinary build (the target is + -- simply absent) and an INPUT in a tool sub-build (the target is + -- what was asked for, so mcpp activates them). Both are needed: + -- `protoc` for libprotoc itself, `upb` because libprotoc's upb + -- generator links the upb runtime — leaving it out fails at LINK + -- with undefined upb_* symbols. + ["protoc"] = { + kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" }, + }, + }, -- protobuf's public headers #include "absl/…" directly, so Abseil is -- part of this package's interface, not an implementation detail. deps = { ["compat.abseil"] = "20250512.1" }, features = { + -- #355: libprotoc — the protobuf COMPILER library, which the `protoc` + -- target links. 138 TUs, transcribed from upstream's own + -- `src/file_lists.cmake` `libprotoc_srcs` (not hand-picked), and with + -- ZERO overlap against the runtime source set above: importer.cc and + -- parser.cc are already there. + -- + -- Off by default, and that is the whole point — a consumer that only + -- links the protobuf runtime must not compile these. + ["protoc"] = { + sources = { + "*/src/google/protobuf/compiler/code_generator.cc", + "*/src/google/protobuf/compiler/code_generator_lite.cc", + "*/src/google/protobuf/compiler/command_line_interface.cc", + "*/src/google/protobuf/compiler/cpp/enum.cc", + "*/src/google/protobuf/compiler/cpp/extension.cc", + "*/src/google/protobuf/compiler/cpp/field.cc", + "*/src/google/protobuf/compiler/cpp/field_chunk.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/map_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/message_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/string_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc", + "*/src/google/protobuf/compiler/cpp/file.cc", + "*/src/google/protobuf/compiler/cpp/generator.cc", + "*/src/google/protobuf/compiler/cpp/helpers.cc", + "*/src/google/protobuf/compiler/cpp/ifndef_guard.cc", + "*/src/google/protobuf/compiler/cpp/message.cc", + "*/src/google/protobuf/compiler/cpp/message_layout_helper.cc", + "*/src/google/protobuf/compiler/cpp/namespace_printer.cc", + "*/src/google/protobuf/compiler/cpp/parse_function_generator.cc", + "*/src/google/protobuf/compiler/cpp/service.cc", + "*/src/google/protobuf/compiler/cpp/tracker.cc", + "*/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc", + "*/src/google/protobuf/compiler/csharp/csharp_enum.cc", + "*/src/google/protobuf/compiler/csharp/csharp_enum_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_field_base.cc", + "*/src/google/protobuf/compiler/csharp/csharp_generator.cc", + "*/src/google/protobuf/compiler/csharp/csharp_helpers.cc", + "*/src/google/protobuf/compiler/csharp/csharp_map_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_message.cc", + "*/src/google/protobuf/compiler/csharp/csharp_message_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_primitive_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_message_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc", + "*/src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc", + "*/src/google/protobuf/compiler/csharp/names.cc", + "*/src/google/protobuf/compiler/java/context.cc", + "*/src/google/protobuf/compiler/java/doc_comment.cc", + "*/src/google/protobuf/compiler/java/field_common.cc", + "*/src/google/protobuf/compiler/java/file.cc", + "*/src/google/protobuf/compiler/java/full/enum.cc", + "*/src/google/protobuf/compiler/java/full/enum_field.cc", + "*/src/google/protobuf/compiler/java/full/extension.cc", + "*/src/google/protobuf/compiler/java/full/generator_factory.cc", + "*/src/google/protobuf/compiler/java/full/make_field_gens.cc", + "*/src/google/protobuf/compiler/java/full/map_field.cc", + "*/src/google/protobuf/compiler/java/full/message.cc", + "*/src/google/protobuf/compiler/java/full/message_builder.cc", + "*/src/google/protobuf/compiler/java/full/message_field.cc", + "*/src/google/protobuf/compiler/java/full/primitive_field.cc", + "*/src/google/protobuf/compiler/java/full/service.cc", + "*/src/google/protobuf/compiler/java/full/string_field.cc", + "*/src/google/protobuf/compiler/java/generator.cc", + "*/src/google/protobuf/compiler/java/helpers.cc", + "*/src/google/protobuf/compiler/java/internal_helpers.cc", + "*/src/google/protobuf/compiler/java/java_features.pb.cc", + "*/src/google/protobuf/compiler/java/lite/enum.cc", + "*/src/google/protobuf/compiler/java/lite/enum_field.cc", + "*/src/google/protobuf/compiler/java/lite/extension.cc", + "*/src/google/protobuf/compiler/java/lite/generator_factory.cc", + "*/src/google/protobuf/compiler/java/lite/make_field_gens.cc", + "*/src/google/protobuf/compiler/java/lite/map_field.cc", + "*/src/google/protobuf/compiler/java/lite/message.cc", + "*/src/google/protobuf/compiler/java/lite/message_builder.cc", + "*/src/google/protobuf/compiler/java/lite/message_field.cc", + "*/src/google/protobuf/compiler/java/lite/primitive_field.cc", + "*/src/google/protobuf/compiler/java/lite/string_field.cc", + "*/src/google/protobuf/compiler/java/message_serialization.cc", + "*/src/google/protobuf/compiler/java/name_resolver.cc", + "*/src/google/protobuf/compiler/java/names.cc", + "*/src/google/protobuf/compiler/java/shared_code_generator.cc", + "*/src/google/protobuf/compiler/kotlin/field.cc", + "*/src/google/protobuf/compiler/kotlin/file.cc", + "*/src/google/protobuf/compiler/kotlin/generator.cc", + "*/src/google/protobuf/compiler/kotlin/message.cc", + "*/src/google/protobuf/compiler/objectivec/enum.cc", + "*/src/google/protobuf/compiler/objectivec/enum_field.cc", + "*/src/google/protobuf/compiler/objectivec/extension.cc", + "*/src/google/protobuf/compiler/objectivec/field.cc", + "*/src/google/protobuf/compiler/objectivec/file.cc", + "*/src/google/protobuf/compiler/objectivec/generator.cc", + "*/src/google/protobuf/compiler/objectivec/helpers.cc", + "*/src/google/protobuf/compiler/objectivec/import_writer.cc", + "*/src/google/protobuf/compiler/objectivec/line_consumer.cc", + "*/src/google/protobuf/compiler/objectivec/map_field.cc", + "*/src/google/protobuf/compiler/objectivec/message.cc", + "*/src/google/protobuf/compiler/objectivec/message_field.cc", + "*/src/google/protobuf/compiler/objectivec/names.cc", + "*/src/google/protobuf/compiler/objectivec/oneof.cc", + "*/src/google/protobuf/compiler/objectivec/primitive_field.cc", + "*/src/google/protobuf/compiler/objectivec/tf_decode_data.cc", + "*/src/google/protobuf/compiler/php/names.cc", + "*/src/google/protobuf/compiler/php/php_generator.cc", + "*/src/google/protobuf/compiler/plugin.cc", + "*/src/google/protobuf/compiler/plugin.pb.cc", + "*/src/google/protobuf/compiler/python/generator.cc", + "*/src/google/protobuf/compiler/python/helpers.cc", + "*/src/google/protobuf/compiler/python/pyi_generator.cc", + "*/src/google/protobuf/compiler/retention.cc", + "*/src/google/protobuf/compiler/ruby/rbs_generator.cc", + "*/src/google/protobuf/compiler/ruby/ruby_generator.cc", + "*/src/google/protobuf/compiler/rust/accessors/accessor_case.cc", + "*/src/google/protobuf/compiler/rust/accessors/accessors.cc", + "*/src/google/protobuf/compiler/rust/accessors/default_value.cc", + "*/src/google/protobuf/compiler/rust/accessors/map.cc", + "*/src/google/protobuf/compiler/rust/accessors/repeated_field.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_cord.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_message.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_string.cc", + "*/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc", + "*/src/google/protobuf/compiler/rust/accessors/with_presence.cc", + "*/src/google/protobuf/compiler/rust/context.cc", + "*/src/google/protobuf/compiler/rust/crate_mapping.cc", + "*/src/google/protobuf/compiler/rust/enum.cc", + "*/src/google/protobuf/compiler/rust/extension.cc", + "*/src/google/protobuf/compiler/rust/generator.cc", + "*/src/google/protobuf/compiler/rust/message.cc", + "*/src/google/protobuf/compiler/rust/naming.cc", + "*/src/google/protobuf/compiler/rust/oneof.cc", + "*/src/google/protobuf/compiler/rust/relative_path.cc", + "*/src/google/protobuf/compiler/rust/rust_field_type.cc", + "*/src/google/protobuf/compiler/rust/rust_keywords.cc", + "*/src/google/protobuf/compiler/rust/upb_helpers.cc", + "*/src/google/protobuf/compiler/subprocess.cc", + "*/src/google/protobuf/compiler/versions.cc", + "*/src/google/protobuf/compiler/zip_writer.cc", + "*/upb_generator/common.cc", + "*/upb_generator/common/names.cc", + "*/upb_generator/file_layout.cc", + "*/upb_generator/minitable/names.cc", + "*/upb_generator/minitable/names_internal.cc", + "*/upb_generator/plugin.cc", + }, + }, -- GzipInputStream / GzipOutputStream. io/gzip_stream.cc is wrapped -- head-to-toe in `#if HAVE_ZLIB`, so by default it compiles to an -- empty TU and the package carries no zlib dependency at all; @@ -319,6 +501,35 @@ package = { -- here each package carries its own compile flags, so it has to be -- stated. No extra import libs: -ladvapi32 arrives with abseil. cxxflags = { "-DNOMINMAX", "-DWIN32_LEAN_AND_MEAN", "-D_CRT_SECURE_NO_WARNINGS" }, + + -- NO `protoc` TARGET ON WINDOWS — a platform `targets` replaces the + -- top-level one, so this drops the tool while keeping the library. + -- + -- Not a protobuf problem and not a flags problem: the tool SUB-BUILD + -- fails there. In the same CI run, tests/examples/protobuf, + -- protobuf-upb and protobuf-gzip all pass on windows — the very same + -- abseil + protobuf sources, built as an ordinary dependency. Only + -- the sub-build dies, and only on three abseil TUs whose `.ddi` scan + -- outputs never appear: + -- + -- error: building host tool 'compat.protobuf:protoc' failed + -- error: cannot read 'obj/compat_abseil/…/absl/time/internal/test_util.cc.ddi' + -- …/cctz/src/time_zone_posix.cc.ddi, …/cctz/src/zone_info_source.cc.ddi + -- + -- It is NOT path length (MAX_PATH was the obvious guess and it is + -- wrong: those three relative paths are 31/46/47 chars, while + -- absl/container/internal/hashtablez_sampler_force_weak_definition.cc + -- at 67 compiles fine in the same sub-build). The sub-build's inner + -- ninja output is summarized, so the underlying scan error is not in + -- the log and the cause is UNKNOWN. + -- + -- Declaring the target on a platform where it cannot be built would + -- hand users a failure with no explanation. Left off until the + -- sub-build issue is diagnosed on a windows host; nothing else about + -- this descriptor is windows-gated. + targets = { + ["protobuf"] = { kind = "lib" }, + }, }, }, } diff --git a/tests/examples/protobuf-protoc/build.mcpp b/tests/examples/protobuf-protoc/build.mcpp new file mode 100644 index 00000000..9a0458e7 --- /dev/null +++ b/tests/examples/protobuf-protoc/build.mcpp @@ -0,0 +1,74 @@ +// Generate inventory.pb.{h,cc} with the protoc this build produced. +// +// The work is DECLARED, not done here: `mcpp::action` makes it an edge in the +// build graph, so it re-runs exactly when the .proto changes and a failure is +// attributed to the edge rather than to "build.mcpp exited 1". +#include +#include + +import mcpp; + +namespace fs = std::filesystem; + +// protoc does NOT embed the well-known types. `import +// "google/protobuf/timestamp.proto"` is read from disk like any other import, +// and the files ship inside the protobuf package this project already depends +// on. Probe for the directory that actually contains descriptor.proto instead +// of hardcoding the tarball's wrap-directory name, which is a packaging +// artifact and not part of any contract. +static std::string well_known_types_dir() { + const std::string base = mcpp::dep_dir("protobuf"); + if (base.empty()) return {}; + std::error_code ec; + for (const auto& entry : fs::directory_iterator(base, ec)) { + const fs::path src = entry.path() / "src"; + if (fs::exists(src / "google" / "protobuf" / "descriptor.proto", ec)) + return src.generic_string(); + } + return {}; +} + +int main() { + // No `protoc` target on windows (see the descriptor's windows block), so + // there is nothing to declare and no include dir to add. Returning 0 keeps + // the member building; tests/codegen.cpp compiles to a visible skip. + if (std::string(mcpp::target_os()) == "windows") return 0; + + const std::string root = mcpp::manifest_dir(); + const std::string out = mcpp::out_dir(); + + const char* protoc = mcpp::dep_bin("protobuf", "protoc"); + if (!protoc || !*protoc) { + std::fputs("no protoc: declare protobuf = { version = \"35.1\", " + "tools = [\"protoc\"] }\n", stderr); + return 1; + } + + const std::string wkt = well_known_types_dir(); + if (wkt.empty()) { + std::fputs("cannot locate the well-known .proto files in the protobuf " + "package\n", stderr); + return 1; + } + + const std::string proto = root + "/proto/inventory.proto"; + + // The .pb.h is declared alongside the .pb.cc because the test includes it + // and it must therefore be PRODUCED by this edge. mcpp knows a header is + // not a translation unit and keeps it out of the compile set. + mcpp::action gen; + gen.id = "protoc:inventory"; + gen.role = "source"; + gen.description = "protoc -> inventory"; + gen.arg(protoc) + .arg(("-I" + root + "/proto").c_str()) + .arg(("-I" + wkt).c_str()) + .arg(("--cpp_out=" + out).c_str()) + .arg(proto.c_str()) + .input(proto.c_str()) + .output((out + "/inventory.pb.cc").c_str()) + .output((out + "/inventory.pb.h").c_str()) + .submit(); + + mcpp::include_dir(out.c_str()); +} diff --git a/tests/examples/protobuf-protoc/mcpp.toml b/tests/examples/protobuf-protoc/mcpp.toml new file mode 100644 index 00000000..82ae3b1d --- /dev/null +++ b/tests/examples/protobuf-protoc/mcpp.toml @@ -0,0 +1,37 @@ +# protobuf `protoc` target member — the compiler, not the runtime. +# +# The sibling tests/examples/protobuf covers the runtime and deliberately uses +# NO generated code. This member is its complement: every line of the message +# API it touches was emitted, during this build, by a protoc that mcpp built +# from the SAME descriptor that provides the runtime being linked. +# +# That co-provenance is the point. protoc and libprotobuf must agree on the +# generated-code ABI, and a mismatch there is a runtime failure, not a build +# error. Here it is not expressible: `tools = ["protoc"]` makes the tool's +# version the dependency's version. +# +# WINDOWS: linux + macOS only, matching the descriptor — compat.protobuf does +# not declare the `protoc` target on windows, because the tool sub-build fails +# there for reasons not yet diagnosed (see the comment in the descriptor's +# windows block). The dependency below is therefore per-OS, and tests/codegen.cpp +# compiles to a visible skip on windows rather than a test that silently proves +# nothing. +[package] +name = "protobuf-protoc-tests" +version = "0.1.0" +standard = "c++23" + +# One dependency, two roles: `features` shapes what gets LINKED (the runtime), +# `tools` asks for a host binary out of the same package. `protoc` pulls in +# libprotoc's 138 TUs and needs `upb` for the upb generator's runtime — the +# descriptor's `required_features` states that, so asking for the tool is +# enough and this manifest does not have to know it. +[target.'cfg(linux)'.dependencies.compat] +protobuf = { version = "35.1", tools = ["protoc"] } + +[target.'cfg(macos)'.dependencies.compat] +protobuf = { version = "35.1", tools = ["protoc"] } + +# The runtime alone, so the member still builds and links something real here. +[target.'cfg(windows)'.dependencies.compat] +protobuf = "35.1" diff --git a/tests/examples/protobuf-protoc/proto/inventory.proto b/tests/examples/protobuf-protoc/proto/inventory.proto new file mode 100644 index 00000000..edd57967 --- /dev/null +++ b/tests/examples/protobuf-protoc/proto/inventory.proto @@ -0,0 +1,37 @@ +// Small on purpose, but it exercises the generator features that break first +// when protoc and the linked runtime disagree: nested messages, an enum, a +// repeated message field, a map, oneof, and a well-known-type import. +syntax = "proto3"; + +package inventory; + +import "google/protobuf/timestamp.proto"; + +enum Grade { + GRADE_UNKNOWN = 0; + GRADE_A = 1; + GRADE_B = 2; +} + +message Item { + string sku = 1; + int32 quantity = 2; + Grade grade = 3; + + message Dimensions { + double width = 1; + double height = 2; + } + Dimensions dimensions = 4; + + oneof source { + string supplier = 5; + string warehouse = 6; + } +} + +message Inventory { + repeated Item items = 1; + map totals_by_grade = 2; + google.protobuf.Timestamp updated_at = 3; +} diff --git a/tests/examples/protobuf-protoc/tests/codegen.cpp b/tests/examples/protobuf-protoc/tests/codegen.cpp new file mode 100644 index 00000000..d70c0f71 --- /dev/null +++ b/tests/examples/protobuf-protoc/tests/codegen.cpp @@ -0,0 +1,120 @@ +// Behavioral test for compat.protobuf's `protoc` target. +// +// Every type used here was emitted DURING THIS BUILD by a protoc that mcpp +// compiled from the same package that provides the runtime being linked. The +// test therefore asserts the thing that actually matters about a code +// generator shipped as a dependency: that its output and the runtime agree. +// +// What it drives, and what would break first on a generator/runtime mismatch: +// +// nested message + accessors generated_message_reflection.cc +// enum generated_enum_util.cc +// repeated message field repeated_ptr_field.cc +// map field map_field.cc +// oneof the generated case() discriminator +// well-known type import timestamp.pb.cc (proves the -I resolved) +// serialize -> parse round trip wire_format_lite.cc, parse_context.cc +// reflection over generated msg descriptor.cc against the generated pool +// +// Returns non-zero on any mismatch. +#include +#include + +// compat.protobuf declares no `protoc` target on windows, so nothing was +// generated and there is nothing to assert. A loud skip beats a test that +// passes without exercising anything. +#ifdef _WIN32 +int main() { + std::puts("skipped: compat.protobuf has no protoc target on windows"); + return 0; +} +#else + +#include "google/protobuf/util/time_util.h" + +#include "inventory.pb.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char* what) { + if (!ok) { + std::fprintf(stderr, "FAIL: %s\n", what); + ++failures; + } +} + +} // namespace + +int main() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + inventory::Inventory inv; + + inventory::Item* widget = inv.add_items(); + widget->set_sku("WIDGET-1"); + widget->set_quantity(7); + widget->set_grade(inventory::GRADE_A); + widget->mutable_dimensions()->set_width(2.5); + widget->mutable_dimensions()->set_height(4.0); + widget->set_supplier("acme"); + + inventory::Item* gizmo = inv.add_items(); + gizmo->set_sku("GIZMO-2"); + gizmo->set_quantity(3); + gizmo->set_grade(inventory::GRADE_B); + gizmo->set_warehouse("east"); + + (*inv.mutable_totals_by_grade())["A"] = 7; + (*inv.mutable_totals_by_grade())["B"] = 3; + + // The well-known type. Reaching this line at all proves protoc resolved + // the import, and setting it proves timestamp.pb.cc is in the runtime. + *inv.mutable_updated_at() = + google::protobuf::util::TimeUtil::SecondsToTimestamp(1735689600); + + std::string wire; + check(inv.SerializeToString(&wire), "serialize"); + check(!wire.empty(), "wire is non-empty"); + + inventory::Inventory back; + check(back.ParseFromString(wire), "parse"); + + check(back.items_size() == 2, "two items survived the round trip"); + check(back.items(0).sku() == "WIDGET-1", "item 0 sku"); + check(back.items(0).quantity() == 7, "item 0 quantity"); + check(back.items(0).grade() == inventory::GRADE_A, "item 0 enum"); + check(back.items(0).dimensions().width() == 2.5, "nested message field"); + check(back.items(0).source_case() == inventory::Item::kSupplier, + "oneof discriminator (supplier)"); + check(back.items(0).supplier() == "acme", "oneof value"); + check(back.items(1).source_case() == inventory::Item::kWarehouse, + "oneof discriminator (warehouse)"); + check(back.totals_by_grade().size() == 2, "map size"); + check(back.totals_by_grade().at("A") == 7, "map lookup"); + check(google::protobuf::util::TimeUtil::TimestampToSeconds( + back.updated_at()) == 1735689600, + "well-known Timestamp round trip"); + + // Reflection over the generated pool: the descriptor protoc emitted has to + // describe the C++ class it emitted beside it. + const google::protobuf::Descriptor* d = inventory::Item::descriptor(); + check(d != nullptr && d->full_name() == "inventory.Item", + "descriptor full name"); + check(d != nullptr && d->FindFieldByName("sku") != nullptr, + "descriptor knows the sku field"); + check(d != nullptr && d->oneof_decl_count() == 1, + "descriptor knows the oneof"); + + google::protobuf::ShutdownProtobufLibrary(); + + if (failures != 0) { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + std::puts("protoc-generated code round-trips against the linked runtime"); + return 0; +} + +#endif // _WIN32 diff --git a/tests/member-timings.tsv b/tests/member-timings.tsv new file mode 100644 index 00000000..2c8a912e --- /dev/null +++ b/tests/member-timings.tsv @@ -0,0 +1,185 @@ +linux abseil 111 +linux archive 49 +linux asio-module 37 +linux asio-ssl 49 +linux boost-ext.ut 30 +linux build-mcpp 28 +linux c-ares 28 +linux catch2 81 +linux catch2-main 53 +linux catch2-v2 13 +linux catch2-v2-main 15 +linux cjson 2 +linux core 67 +linux curl 26 +linux eigen 25 +linux eui-neo 151 +linux eui-neo-app-main 140 +linux eui-neo-markdown 143 +linux eui-neo-sdl2 119 +linux eui-neo-vulkan 154 +linux eui-neo-window 128 +linux ffmpeg 288 +linux ffmpeg-module 343 +linux fmtlib.fmt 5 +linux freetype 10 +linux glad 2 +linux godot-cpp 573 +linux godot-cpp-module 549 +linux godot-cpp-module-v10 392 +linux godot-cpp-v10 504 +linux grpc-module 1701 +linux gui-stack 81 +linux imgui 6 +linux imgui-module 91 +linux imgui-window 91 +linux libpng 6 +linux llamacpp 101 +linux llamacpp-metal 0 +linux magic_enum 4 +linux marzer.tomlplusplus 7 +linux md4c 3 +linux nlohmann.json 10 +linux openblas 0 +linux opencv-module 602 +linux opencv-module-dnn 741 +linux opencv-module-unifont 671 +linux openssl 0 +linux protobuf 259 +linux protobuf-gzip 158 +linux protobuf-protoc 923 +linux protobuf-upb 263 +linux re2 15 +linux sdl2 63 +linux spdlog 9 +linux spdlog-compiled 13 +linux tinyhttps 13 +linux tray 3 +linux vulkan 13 +linux websocket 19 +linux websocket-features 23 +linux yyjson 3 +macos abseil 75 +macos archive 49 +macos asio-module 30 +macos asio-ssl 78 +macos boost-ext.ut 27 +macos build-mcpp 33 +macos c-ares 27 +macos catch2 58 +macos catch2-main 28 +macos catch2-v2 10 +macos catch2-v2-main 9 +macos cjson 3 +macos core 58 +macos curl 103 +macos eigen 7 +macos eui-neo 52 +macos eui-neo-app-main 57 +macos eui-neo-markdown 76 +macos eui-neo-sdl2 142 +macos eui-neo-vulkan 59 +macos eui-neo-window 53 +macos ffmpeg 148 +macos ffmpeg-module 130 +macos fmtlib.fmt 4 +macos freetype 12 +macos glad 3 +macos godot-cpp 333 +macos godot-cpp-module 256 +macos godot-cpp-module-v10 215 +macos godot-cpp-v10 230 +macos grpc-module 880 +macos gui-stack 1 +macos imgui 4 +macos imgui-module 1 +macos imgui-window 1 +macos libpng 7 +macos llamacpp 66 +macos llamacpp-metal 75 +macos magic_enum 3 +macos marzer.tomlplusplus 5 +macos md4c 4 +macos nlohmann.json 9 +macos openblas 1 +macos opencv-module 198 +macos opencv-module-dnn 399 +macos opencv-module-unifont 304 +macos openssl 1 +macos protobuf 100 +macos protobuf-gzip 87 +macos protobuf-protoc 458 +macos protobuf-upb 143 +macos re2 8 +macos sdl2 22 +macos spdlog 5 +macos spdlog-compiled 6 +macos tinyhttps 13 +macos tray 4 +macos vulkan 7 +macos websocket 12 +macos websocket-features 12 +macos yyjson 6 +# — measured, run 31034885938 +# refresh: download the member-timings artifact from a full run and replace this file +windows abseil 133 +windows archive 76 +windows asio-module 37 +windows asio-ssl 24 +windows boost-ext.ut 40 +windows build-mcpp 27 +windows c-ares 33 +windows catch2 83 +windows catch2-main 62 +windows catch2-v2 12 +windows catch2-v2-main 12 +windows cjson 3 +windows core 98 +windows curl 31 +windows eigen 10 +windows eui-neo 93 +windows eui-neo-app-main 119 +windows eui-neo-markdown 84 +windows eui-neo-sdl2 150 +windows eui-neo-vulkan 84 +windows eui-neo-window 74 +windows ffmpeg 454 +windows ffmpeg-module 532 +windows fmtlib.fmt 5 +windows freetype 15 +windows glad 2 +windows godot-cpp 742 +windows godot-cpp-module 636 +windows godot-cpp-module-v10 684 +windows godot-cpp-v10 566 +windows grpc-module 1 +windows gui-stack 1 +windows imgui 6 +windows imgui-module 0 +windows imgui-window 0 +windows libpng 10 +windows llamacpp 125 +windows llamacpp-metal 1 +windows magic_enum 4 +windows marzer.tomlplusplus 6 +windows md4c 3 +windows nlohmann.json 10 +windows openblas 11 +windows opencv-module 747 +windows opencv-module-dnn 1174 +windows opencv-module-unifont 1 +windows openssl 1 +windows protobuf 215 +windows protobuf-gzip 141 +windows protobuf-protoc 227 +windows protobuf-upb 251 +windows re2 15 +windows sdl2 46 +windows spdlog 6 +windows spdlog-compiled 14 +windows tinyhttps 14 +windows tray 2 +windows vulkan 7 +windows websocket 24 +windows websocket-features 26 +windows yyjson 2 diff --git a/tests/plan_shards.lua b/tests/plan_shards.lua new file mode 100644 index 00000000..e92f8665 --- /dev/null +++ b/tests/plan_shards.lua @@ -0,0 +1,168 @@ +#!/usr/bin/env lua5.4 +-- plan_shards.lua — decide which members run on which shard. +-- +-- lua5.4 tests/plan_shards.lua [member...] +-- +-- Prints this shard's members, space-separated. With no member arguments it +-- reads the whole workspace from mcpp.toml. +-- +-- WHY NOT ROUND-ROBIN BY POSITION +-- +-- The first version split `i % count`, which knows nothing about how long +-- anything takes; balance was luck. Measured on the real workspace it put +-- three heavyweights (ffmpeg, llamacpp-metal, opencv-module-unifont) on one +-- shard and none on three others — and wall-clock is the SLOWEST shard, so the +-- idle ones bought nothing. +-- +-- Two things decide the split here instead: +-- +-- 1. MEASURED TIME (tests/member-timings.tsv, produced by CI). Longest- +-- Processing-Time first: sort descending, put each member on the shard +-- with the least load so far. LPT is within 4/3 of optimal for this +-- problem, and optimal is not worth more than that here. +-- 2. DEPENDENCY AFFINITY, as the tie-break. Two members that share a +-- dependency build it once if they land on the same shard and twice if +-- they do not — shards do not share a build cache, only a run does. So +-- among shards whose load is close, prefer the one already holding +-- members with overlapping dependencies. +-- +-- Missing timing → the median, so a newly added member is neither assumed +-- free nor assumed huge. No table at all → falls back to round-robin, which +-- is worse but never wrong. +-- +-- Measured on the real workspace (linux, 3 shards), LPT against round-robin: +-- +-- round-robin 4158 / 3027 / 2822 slowest 4158s +-- LPT 3706 / 3152 / 3149 slowest 3706s +-- +-- 452s off the wall-clock, and the spread drops from 47% to 15%. +-- +-- There is a floor no split can beat: the single slowest member. grpc-module +-- alone is 1701s, so linux cannot finish faster than that however many shards +-- there are — which is the number to look at before adding more. + +local platform = arg[1] or error("usage: plan_shards.lua [members...]") +local shardIndex = tonumber(arg[2]) or error("shard index must be a number") +local shardCount = tonumber(arg[3]) or error("shard count must be a number") + +local members = {} +for i = 4, #arg do members[#members + 1] = arg[i] end + +local function read_file(path) + local f = io.open(path, "r"); if not f then return nil end + local s = f:read("a"); f:close(); return s +end + +-- Whole workspace, from the manifest rather than the directory: a member that +-- exists on disk but is not registered must not be tested. Both filters below +-- matter — mcpp.toml's prose mentions `tests/examples/` too, which yields an +-- empty name, and a name that only appears in a comment is not a member. +if #members == 0 then + local toml = read_file("mcpp.toml") or error("cannot read mcpp.toml") + local seen = {} + for name in toml:gmatch("tests/examples/([A-Za-z0-9._%-]+)") do + if name ~= "" and not seen[name] then + local probe = io.open("tests/examples/" .. name .. "/mcpp.toml", "r") + if probe then probe:close(); seen[name] = true; members[#members + 1] = name end + end + end + table.sort(members) +end + +if shardCount <= 1 then + print(table.concat(members, " ")) + return +end + +-- ── measured times ──────────────────────────────────────────────────────── +-- Format: \t\t +local times, samples = {}, {} +local tsv = read_file("tests/member-timings.tsv") +if tsv then + for line in tsv:gmatch("[^\n]+") do + if not line:match("^#") then + local p, m, s = line:match("^(%S+)\t(%S+)\t(%d+)") + if p == platform and m then + times[m] = tonumber(s) + samples[#samples + 1] = tonumber(s) + end + end + end +end + +local median = 60 +if #samples > 0 then + table.sort(samples) + median = samples[math.ceil(#samples / 2)] +end + +-- ── dependency signature, for affinity ──────────────────────────────────── +local function deps_of(member) + local toml = read_file("tests/examples/" .. member .. "/mcpp.toml") + if not toml then return {} end + local set = {} + -- The package names a member depends on. Deliberately crude: exact + -- accuracy is not needed, only "do these two pull the same big things". + for name in toml:gmatch("\n%s*([A-Za-z][A-Za-z0-9._%-]*)%s*=") do + if name ~= "name" and name ~= "version" and name ~= "standard" + and name ~= "sources" and name ~= "kind" and name ~= "main" + and name ~= "description" and name ~= "license" then + set[name] = true + end + end + return set +end + +local depsCache = {} +local function deps(member) + if depsCache[member] == nil then depsCache[member] = deps_of(member) end + return depsCache[member] +end + +-- ── LPT with affinity tie-break ─────────────────────────────────────────── +local ordered = {} +for _, m in ipairs(members) do ordered[#ordered + 1] = m end +table.sort(ordered, function(a, b) + local ta, tb = times[a] or median, times[b] or median + if ta ~= tb then return ta > tb end + return a < b -- deterministic across machines +end) + +local shards = {} +for i = 0, shardCount - 1 do shards[i] = { load = 0, members = {}, deps = {} } end + +for _, m in ipairs(ordered) do + local cost = times[m] or median + local best, bestLoad, bestAffinity = nil, nil, -1 + for i = 0, shardCount - 1 do + local s = shards[i] + -- Affinity only breaks near-ties: a shard 15% lighter always wins, + -- because balance is what wall-clock actually measures. + local affinity = 0 + for d in pairs(deps(m)) do if s.deps[d] then affinity = affinity + 1 end end + if best == nil then + best, bestLoad, bestAffinity = i, s.load, affinity + else + local margin = math.max(bestLoad, s.load) * 0.15 + if s.load < bestLoad - margin then + best, bestLoad, bestAffinity = i, s.load, affinity + elseif math.abs(s.load - bestLoad) <= margin and affinity > bestAffinity then + best, bestLoad, bestAffinity = i, s.load, affinity + end + end + end + local s = shards[best] + s.load = s.load + cost + s.members[#s.members + 1] = m + for d in pairs(deps(m)) do s.deps[d] = true end +end + +if os.getenv("PLAN_SHARDS_DEBUG") then + for i = 0, shardCount - 1 do + io.stderr:write(string.format("shard %d: load=%ds n=%d %s\n", + i, shards[i].load, #shards[i].members, + table.concat(shards[i].members, " "))) + end +end + +print(table.concat(shards[shardIndex] and shards[shardIndex].members or {}, " ")) diff --git a/tests/run_members.sh b/tests/run_members.sh new file mode 100755 index 00000000..31c1172d --- /dev/null +++ b/tests/run_members.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# run_members.sh — run workspace members one by one, timing each. +# +# The same script CI runs and you run locally, on purpose: a timing table that +# only exists in CI cannot be used while deciding what to optimise, and a local +# harness that differs from CI measures something else. +# +# bash tests/run_members.sh --all +# bash tests/run_members.sh opencv-module protobuf +# bash tests/run_members.sh --all --shard 3/8 +# bash tests/run_members.sh --all --cache local # bypass the package cache +# +# Env: +# MCPP path to the mcpp binary (default: `mcpp` on PATH) +# MCPP_TIMINGS where to append `\t\t` rows +# +# Exit status is non-zero if any member failed. The timing table is printed +# regardless — a slow run is worth measuring even when it breaks. +set -u + +MCPP="${MCPP:-mcpp}" +timings="${MCPP_TIMINGS:-}" +cache="" +shard="" +members=() +all=0 + +while [ $# -gt 0 ]; do + case "$1" in + --all) all=1; shift ;; + --shard) shard="$2"; shift 2 ;; + --cache) cache="$2"; shift 2 ;; + --timings) timings="$2"; shift 2 ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + -*) echo "unknown option: $1" >&2; exit 2 ;; + *) members+=("$1"); shift ;; + esac +done + +# `--all` reads the workspace manifest rather than the directory, so a member +# that exists on disk but is not registered is not silently tested. +# +# Both filters below are load-bearing. mcpp.toml's PROSE mentions the path too +# — line 3 says "tests/examples/ — each consumes this repo's own packages", +# which this grep matches with an empty tail, and `mcpp test -p ""` is not a +# useful thing to run. Requiring a real directory also means a name that only +# appears in a comment (`tests/examples/asio-ssl` is discussed in one) cannot +# turn into a phantom member. +if [ "$all" = 1 ]; then + while IFS= read -r m; do + [ -n "$m" ] || continue + [ -d "tests/examples/$m" ] || continue + members+=("$m") + done < <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml \ + | sed 's|tests/examples/||' | sort -u) +fi + +if [ "${#members[@]}" -eq 0 ]; then + echo "no members selected — pass names or --all" >&2 + exit 2 +fi + +# --shard N/M keeps every M-th member starting at N. Round-robin by position, +# which is what separates adjacent expensive members (opencv-module, +# -dnn, -unifont) onto different runners. +if [ -n "$shard" ]; then + idx=${shard%%/*} + cnt=${shard##*/} + picked=() + i=0 + for m in "${members[@]}"; do + [ $((i % cnt)) -eq "$idx" ] && picked+=("$m") + i=$((i + 1)) + done + members=("${picked[@]+"${picked[@]}"}") + echo "shard $idx/$cnt -> ${#members[@]} member(s)" +fi + +[ -n "$cache" ] && export MCPP_BUILD_CACHE="$cache" +echo "cache mode: ${MCPP_BUILD_CACHE:-global (default)}" + +rows=$(mktemp) +trap 'rm -f "$rows"' EXIT +rc=0 + +for m in "${members[@]}"; do + echo "::group::mcpp test -p $m" + t0=$(date +%s) + if "$MCPP" test -p "$m"; then status=ok; else status=FAIL; rc=1; fi + t1=$(date +%s) + echo "::endgroup::" + printf '%s\t%s\t%s\n' "$((t1 - t0))" "$m" "$status" >> "$rows" + printf ' %-34s %5ss %s\n' "$m" "$((t1 - t0))" "$status" +done + +[ -n "$timings" ] && cat "$rows" >> "$timings" + +echo +echo "── slowest members ──────────────────────────────────────────" +total=$(awk -F'\t' '{s += $1} END {print s+0}' "$rows") +sort -rn "$rows" | head -15 | awk -F'\t' -v tot="$total" ' + { pct = tot > 0 ? ($1 * 100 / tot) : 0 + printf " %6ss %5.1f%% %-34s %s\n", $1, pct, $2, $3 }' +echo " ────────" +printf ' %6ss total across %s member(s)\n' "$total" "${#members[@]}" + +exit "$rc"