From 46f41c40bc20e62b5dc16fbab1cce3596a0738df Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Wed, 5 Aug 2026 17:02:23 +0800 Subject: [PATCH 01/10] feat(protobuf): expose protoc as a host tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit compat.protobuf shipped the libprotobuf runtime and nothing else, so a consumer that needed generated code had to find a protoc somewhere and guarantee by hand that it matched the runtime being linked. That mismatch is a RUNTIME failure — the classic protobuf footgun — and it is the whole reason Conan carries a `protobuf/` placeholder and xmake deletes protoc outright when cross-compiling. mcpp 2026.8.5.1 makes it inexpressible: a `kind = "bin"` target in the descriptor, asked for by the consumer as `tools = ["protoc"]`, is built for the BUILD machine out of the same package the consumer links. The tool's version IS the dependency's version, and `--target` does not change that because a code generator has to run here. - pkgs/c/compat.protobuf.lua: add the `protoc` target (main = "*/src/google/protobuf/compiler/main.cc") plus a `protoc` feature carrying the 138 entries of upstream's own `libprotoc_srcs` from src/file_lists.cmake — zero overlap with the libprotobuf source set, and no configure step (the tree holds no .h.in/.cmake.in). The target declares required_features = { "protoc", "upb" }: libprotoc's upb generator links the upb runtime, and without it the link fails on missing `upb_*` symbols, so the descriptor states the constraint instead of leaving consumers to discover it. Consumers who only link the runtime compile none of this. - tests/examples/protobuf-protoc: the complement of tests/examples/protobuf. That member deliberately uses NO generated code; this one is generated code end to end — nested messages, enum, repeated field, map, oneof, a well-known-type import and reflection over the generated pool, serialized and parsed back. Verified locally: the four files this toolchain generates for grpc-m's helloworld.proto are BYTE-IDENTICAL to the officially generated stubs checked into that repo. - CI pin -> 2026.8.5.1. This is a floor, not a routine bump: `tools = [...]` does not parse before it ("tools must be a string, inline dep table, or nested table"), confirmed against 2026.8.3.3. - index.toml: min_mcpp deliberately UNCHANGED. It is a gate — an older client failing it cannot open the index at all (mcpp#349) — and this change does not make any descriptor unreadable: 2026.8.3.3 still parses compat.protobuf with an empty unknown_keys. Only the consumer spelling needs the newer mcpp, and that lives in a consumer's own mcpp.toml. latest_mcpp, an advisory hint with no gate behaviour, tracks what CI validates against. - docs: shape H (host tool provider) in package-types.md + zh, README rows. Includes the one sharp edge: protoc does not embed the well-known types, so `import "google/protobuf/timestamp.proto"` needs an -I derived from mcpp::dep_dir("protobuf"). --- .github/workflows/validate.yml | 31 ++- README.md | 2 +- README.zh-CN.md | 2 +- docs/package-types.md | 45 ++++ docs/zh/package-types.md | 43 ++++ index.toml | 12 +- mcpp.toml | 1 + pkgs/c/compat.protobuf.lua | 200 +++++++++++++++++- tests/examples/protobuf-protoc/build.mcpp | 69 ++++++ tests/examples/protobuf-protoc/mcpp.toml | 23 ++ .../protobuf-protoc/proto/inventory.proto | 37 ++++ .../protobuf-protoc/tests/codegen.cpp | 108 ++++++++++ 12 files changed, 554 insertions(+), 19 deletions(-) create mode 100644 tests/examples/protobuf-protoc/build.mcpp create mode 100644 tests/examples/protobuf-protoc/mcpp.toml create mode 100644 tests/examples/protobuf-protoc/proto/inventory.proto create mode 100644 tests/examples/protobuf-protoc/tests/codegen.cpp diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e96666b4..b292c934 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -14,6 +14,15 @@ on: workflow_dispatch: env: + # 2026.8.5.1 is a FLOOR, not a routine bump: `tools = [...]` — how a consumer + # asks for a dependency's `kind = "bin"` target, which the new + # tests/examples/protobuf-protoc member is built on — does not parse before + # it ("tools must be a string, inline dep table, or nested table"). Note this + # does NOT move index.toml's min_mcpp: exposing compat.protobuf's `protoc` + # target is additive, and 2026.8.3.3 still parses that descriptor with an + # empty unknown_keys. The floor an index publishes decides whether older + # clients keep working at all (mcpp#349), so it moves only when a descriptor + # genuinely stops being readable — which is not the case here. # 2026.8.3.1: on macOS, a global object that touches std::cout during static # init crashes on sight (mcpp#336). Mach-O has no priority-ordered init # section and libc++'s carries no ios_base::Init guard of its own, @@ -88,7 +97,7 @@ env: # 0.0.94 fixed feature-gated `sources` under `mcpp test` (mcpp#218); 0.0.91 # added standard = "c++fly" to the resolver grammar, so c++fly descriptors # get the lint WARN below, not a hard grammar-parse rejection. - MCPP_VERSION: "2026.8.3.3" + MCPP_VERSION: "2026.8.5.1" jobs: lint: @@ -253,21 +262,21 @@ jobs: ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.1" # keep in sync with env.MCPP_VERSION - platform: macos os: macos-15 suffix: macosx-arm64 ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.1" # keep in sync with env.MCPP_VERSION - platform: windows os: windows-latest suffix: windows-x86_64 ext: zip mcpp: bin/mcpp.exe xlings: registry/bin/xlings.exe - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.1" # keep in sync with env.MCPP_VERSION env: MCPP_EFFECTIVE: ${{ matrix.mcpp_version }} steps: @@ -448,9 +457,17 @@ jobs: # every zlib consumer without bzip2 then asks the same key for a # flat obj/compress.o and ninja dies at graph time with # "missing and no known rule to make it". Reproduced both ways round - # on 2026.8.3.3 and filed as mcpp-community/mcpp#344; drop this once - # it lands. `local` still caches the std BMI, which is the expensive - # one — only package entries are bypassed. + # on 2026.8.3.3 and filed as mcpp-community/mcpp#344. `local` still + # caches the std BMI, which is the expensive one — only package + # entries are bypassed. + # + # STATUS: #344 landed in 2026.8.3.4 (per-package Merkle keys that + # cover the consumer-dependent layout), so as of the pin above this + # bypass is no longer required. It is kept here deliberately: + # re-enabling the global package cache changes how EVERY member + # builds, and that deserves its own PR where a failure is + # unambiguous rather than being attributed to whatever else shipped + # alongside it. MCPP_BUILD_CACHE: local run: | "$MCPP" --version diff --git a/README.md b/README.md index 0e65ff2a..4159e6cd 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Two kinds of packages live here: |------|------| | Native module library (Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](pkgs/o/opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) | | C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | +| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | | header-only (with `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | Runtime loader compat (pure sources, sidestepping upstream codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua) (the Khronos loader: `loader/generated/` is checked in, and the assembly path degrades to plain C through `UNKNOWN_FUNCTIONS_SUPPORTED`, so no CMake/Python/assembler is needed; windows deferred) · [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) | | Whole-source direct build + generated config (only where a platform lacks one) | [`compat.curl`](pkgs/c/compat.curl.lua) (win32 uses upstream's checked-in config, unix generates one) · [`compat.sdl2`](pkgs/c/compat.sdl2.lua) (win/mac use upstream's checked-in config; linux generates one and enables X11 by hand) · [`compat.c-ares`](pkgs/c/compat.c-ares.lua) (91 TUs; the release tarball already ships `ares_build.h` and a Windows config, so only `ares_config.h` is snapshotted per OS) | diff --git a/README.zh-CN.md b/README.zh-CN.md index d80c213f..f3693bd8 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -36,7 +36,7 @@ mcpp self config --mirror CN # 切换至国内镜像,默认使用 GLOBAL 上 |------|------| | 原生模块库(Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](pkgs/o/opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)| | C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | +| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | | header-only(含 `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | 运行时 loader compat(纯源码,绕开上游 codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua)(Khronos loader:`loader/generated/` 已签入,汇编路径经 `UNKNOWN_FUNCTIONS_SUPPORTED` 降级为纯 C,故无需 CMake/Python/汇编器;windows 延后)· [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) | | 全源码直编 + 生成 config(仅缺口平台) | [`compat.curl`](pkgs/c/compat.curl.lua)(win32 用上游签入 config,unix 生成) · [`compat.sdl2`](pkgs/c/compat.sdl2.lua)(win/mac 用上游签入 config,linux 生成 + 手工开 X11) · [`compat.c-ares`](pkgs/c/compat.c-ares.lua)(91 TU;release tarball 已自带 `ares_build.h` 与 Windows 配置,故只需按 OS 冻结 `ares_config.h`) | diff --git a/docs/package-types.md b/docs/package-types.md index d348f493..3930a093 100644 --- a/docs/package-types.md +++ b/docs/package-types.md @@ -18,6 +18,7 @@ combined as needed. | **E. Whole-source direct build with a generated config** | upstream generates its config header through configure/CMake; here a snapshot of it lands in `generated_files` | `pkgs/c/compat.libpng.lua`, `compat.curl.lua`, `compat.sdl2.lua`, `compat.ffmpeg.lua` | `generated_files` + `include_dirs` | | **F. Shared-library compat** | has to be the **only** copy of that `.so` in the process (third parties `dlopen` it) | the X11 family such as `pkgs/c/compat.x11.lua`, and `compat.vulkan.lua` (linux) | `targets = { kind = "shared", soname = … }` | | **G. Host runtime adaptation** | things that cannot be vendored, such as drivers — only a symlink farm plus metadata | `pkgs/c/compat.glx-runtime.lua`, `compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` | +| **H. Host tool provider** | the upstream tarball also holds a **code generator** consumers run at build time | `pkgs/c/compat.protobuf.lua` (`protoc`) | a `targets` entry with `kind = "bin"` + `main`, plus `required_features` | For the complete sample index, see the [Reference examples table in the root README](../README.md#reference-examples-lua-descriptors). @@ -200,6 +201,50 @@ Two details that keep biting: - **The closure has to be complete.** A farm holding `libxcb.so.1` but not the `libXau.so.6` it depends on shadows the host copy that would otherwise have resolved, and the executable simply fails to start. +## H. Host tool provider (`compat.protobuf`'s `protoc`) + +Some tarballs hold both a library and the code generator that emits code against it. Declare the generator as a second +target, and consumers ask for it with `tools = [...]` (mcpp 2026.8.5.1+): + +```lua +targets = { + ["protobuf"] = { kind = "lib" }, + ["protoc"] = { kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" } }, +}, +features = { + ["protoc"] = { sources = { … the compiler's own sources … } }, +}, +``` + +```toml +# consumer side — one dependency, two roles +compat.protobuf = { version = "35.1", tools = ["protoc"] } +``` + +mcpp then builds that target **for the build machine** in a nested sub-build and hands the path to the consumer's +`build.mcpp` through `mcpp::dep_bin("protobuf", "protoc")`. This is the whole reason the shape is worth naming: the +tool's version **is** the dependency's version, so a generator/runtime mismatch — a *runtime* failure everywhere else, +and the classic protobuf footgun — is not expressible. Under `mcpp build --target ` the tool is still built for +the host, because a code generator has to run here. + +Four things to get right: + +- **Gate the compiler's sources behind a feature**, and name it in the target's `required_features`. Consumers who only + link the library must not pay for the generator's TUs; consumers who ask for the tool must not have to know which + features it needs. `compat.protobuf`'s `protoc` also requires `upb`, because libprotoc's upb generator links the upb + runtime — get that wrong and it fails at **link** time with missing `upb_*` symbols. +- **Transcribe the source list from upstream**, exactly as for a library — protobuf's 138 entries come from + `libprotoc_srcs` in its own `src/file_lists.cmake`. +- **`main` needs the same `*/` wrap glob as `sources`**; it is expanded the same way. +- **A generator that reads data files at runtime still needs a path to them.** protoc does not embed the well-known + types: `import "google/protobuf/timestamp.proto"` is read from disk. Consumers derive that directory from + `mcpp::dep_dir("protobuf")` — see `tests/examples/protobuf-protoc/build.mcpp`. + +The matching member is `tests/examples/protobuf-protoc`, and it is the complement of `tests/examples/protobuf`: that +one deliberately uses no generated code, this one is generated code end to end. + --- ## The minimal project (`tests/examples//`) diff --git a/docs/zh/package-types.md b/docs/zh/package-types.md index 4ffc00eb..becdcd33 100644 --- a/docs/zh/package-types.md +++ b/docs/zh/package-types.md @@ -16,6 +16,7 @@ A–D 是四种**基础**形态,先按它们判定;E–G 是在基础形态之 | **E. 生成 config 的全源码直编** | 上游用 configure/CMake 生成配置头,此处以 `generated_files` 落一份快照 | `pkgs/c/compat.libpng.lua`、`compat.curl.lua`、`compat.sdl2.lua`、`compat.ffmpeg.lua` | `generated_files` + `include_dirs` | | **F. 共享库 compat** | 必须是**唯一**的那个 `.so`(会被第三方 `dlopen`) | `pkgs/c/compat.x11.lua` 等 X11 家族、`compat.vulkan.lua`(linux) | `targets = { kind = "shared", soname = … }` | | **G. 宿主运行时适配** | 驱动之类无法 vendor 的东西,只做符号链接农场 + 元数据 | `pkgs/c/compat.glx-runtime.lua`、`compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` | +| **H. 宿主工具提供方** | 上游 tarball 里除了库,还带着消费者在构建期要跑的**代码生成器** | `pkgs/c/compat.protobuf.lua`(`protoc`) | `targets` 里一条 `kind = "bin"` + `main`,配 `required_features` | 完整的样例索引见[根 README 的「参考示例」表](../../README.zh-CN.md#参考示例lua-描述符)。 @@ -184,6 +185,48 @@ runtime = { - **闭包必须完整**。农场里有 `libxcb.so.1` 却没有它依赖的 `libXau.so.6`,会遮蔽掉本来能解析的宿主副本,可执行 文件直接起不来。 +## H. 宿主工具提供方(`compat.protobuf` 的 `protoc`) + +有些 tarball 里同时装着一个库,和「针对这个库生成代码」的那个生成器。把生成器声明成第二个 target, +消费者用 `tools = [...]` 索取(mcpp 2026.8.5.1 起): + +```lua +targets = { + ["protobuf"] = { kind = "lib" }, + ["protoc"] = { kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" } }, +}, +features = { + ["protoc"] = { sources = { … 编译器自身的源码 … } }, +}, +``` + +```toml +# 消费者侧 —— 一条依赖,两种角色 +compat.protobuf = { version = "35.1", tools = ["protoc"] } +``` + +mcpp 会在一次嵌套子构建里把这个 target 编成**构建机**的二进制,并把路径经 +`mcpp::dep_bin("protobuf", "protoc")` 交给消费者的 `build.mcpp`。这个形态值得单列的全部理由在于: +工具的版本**就是**那条依赖的版本,于是「生成器与运行时版本错配」——在别处是**运行期**才炸、也正是 +protobuf 最经典的坑——在这里**语法上无法表达**。`mcpp build --target ` 下工具仍为宿主构建, +因为代码生成器必须在本机跑。 + +四个要点: + +- **把编译器的源码关进一个 feature**,并在 target 的 `required_features` 里写明。只链库的消费者不该为 + 生成器的 TU 买单;索取工具的消费者也不该需要知道它要哪些 feature。`compat.protobuf` 的 `protoc` 还 + 必须要 `upb`,因为 libprotoc 的 upb 生成器要链 upb 运行时——搞错了会在**链接期**缺一批 `upb_*` 符号。 +- **源码列表照旧逐条转录自上游**:protobuf 这 138 项来自它自己的 `src/file_lists.cmake` 的 `libprotoc_srcs`。 +- **`main` 和 `sources` 一样需要 `*/` 那层 wrap glob**,展开方式相同。 +- **运行期还要读数据文件的生成器,仍然需要一个路径**。protoc 并不内嵌 well-known types: + `import "google/protobuf/timestamp.proto"` 是从磁盘读的。消费者用 `mcpp::dep_dir("protobuf")` 推出那个 + 目录——见 `tests/examples/protobuf-protoc/build.mcpp`。 + +对应的成员是 `tests/examples/protobuf-protoc`,它与 `tests/examples/protobuf` 互为补集:那个刻意**不用** +任何生成代码,这个从头到尾都是生成代码。 + --- ## 最小工程(`tests/examples//`) diff --git a/index.toml b/index.toml index 2793f7ff..8e64e621 100644 --- a/index.toml +++ b/index.toml @@ -7,5 +7,15 @@ # "floor first, new grammar after" rollout rule mechanically. [index] spec = "1" +# +# These two move independently, and the 2026.8.5.1 CI pin is why. `min_mcpp` is +# a GATE — an older client that fails it cannot open this index at all +# (mcpp#349), so it moves only when a descriptor genuinely stops being readable +# by the older grammar. Exposing compat.protobuf's `protoc` target did not do +# that: 2026.8.3.3 still parses the descriptor with an empty `unknown_keys`. +# What needs 2026.8.5.1 is the CONSUMER spelling `tools = [...]`, which lives +# in a consumer's own mcpp.toml — including this repo's +# tests/examples/protobuf-protoc, hence the CI pin. `latest_mcpp` is an +# advisory hint with no gate behaviour, and tracks what CI validates against. min_mcpp = "2026.8.3.3" -latest_mcpp = "2026.8.3.3" +latest_mcpp = "2026.8.5.1" diff --git a/mcpp.toml b/mcpp.toml index 7683cf23..d6399170 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -47,6 +47,7 @@ members = [ "tests/examples/protobuf", "tests/examples/protobuf-upb", "tests/examples/protobuf-gzip", + "tests/examples/protobuf-protoc", "tests/examples/opencv-module", "tests/examples/opencv-module-dnn", "tests/examples/opencv-module-unifont", diff --git a/pkgs/c/compat.protobuf.lua b/pkgs/c/compat.protobuf.lua index fb5ab549..4ba1d892 100644 --- a/pkgs/c/compat.protobuf.lua +++ b/pkgs/c/compat.protobuf.lua @@ -14,14 +14,22 @@ -- source list covers linux/macosx/windows and the three xpm blocks share -- a single tarball and sha256. -- --- SCOPE — runtime only. This package builds upstream's `libprotobuf` target --- (79 TUs), i.e. what a program that *uses* generated code needs: messages, --- reflection, descriptors, text/JSON formats, the well-known types. It does --- NOT build `libprotoc` (a further 157 TUs) and ships no protoc binary, so it --- does not generate .pb.cc from .proto. Consumers either check in --- protoc-generated sources or build them with the official upstream protoc --- release (protoc-35.1-.zip); wiring that into an mcpp build belongs --- to a build.mcpp step, not to this descriptor. +-- SCOPE — runtime by default, compiler on request. This package builds +-- upstream's `libprotobuf` target (79 TUs) unconditionally: messages, +-- reflection, descriptors, text/JSON formats, the well-known types — what a +-- program that *uses* generated code needs. +-- +-- Since mcpp 2026.8.5.1 it ALSO offers `protoc` as a host tool, behind the +-- `protoc` feature (upstream's `libprotoc`, 138 further TUs). A consumer that +-- only links the runtime compiles none of them: +-- +-- compat.protobuf = { version = "35.1", tools = ["protoc"] } +-- +-- That replaces the old advice of "check in protoc output, or fetch the +-- official protoc-35.1-.zip and keep its version in step by hand". +-- Keeping it in step by hand is precisely the failure this removes: a protoc +-- that disagrees with the runtime fails at RUNTIME, and here the tool's +-- version IS this package's version, so the mismatch cannot be expressed. -- -- Version numbering follows upstream verbatim: `35.1` is the protobuf release, -- and it is what gRPC 1.83.0 pins (its third_party/protobuf submodule is @@ -194,13 +202,187 @@ package = { "*/third_party/utf8_range/utf8_range.c", }, - targets = { ["protobuf"] = { kind = "lib" } }, + targets = { + ["protobuf"] = { kind = "lib" }, + -- #355 (mcpp 2026.8.5.1+): protoc as a HOST tool a consumer can ask + -- for, so it never has to supply a matching one by hand: + -- + -- compat.protobuf = { version = "35.1", tools = ["protoc"] } + -- + -- The version axis is what matters here. protoc generating code for + -- a DIFFERENT protobuf runtime than the one being linked fails at + -- RUNTIME, not at compile time, and is the single nastiest thing + -- about hand-managed protobuf codegen. Because the tool's version + -- IS this package's version, that mismatch is not expressible. + -- + -- `required_features` is a GATE in an ordinary build (the target is + -- simply absent) and an INPUT in a tool sub-build (the target is + -- what was asked for, so mcpp activates them). Both are needed: + -- `protoc` for libprotoc itself, `upb` because libprotoc's upb + -- generator links the upb runtime — leaving it out fails at LINK + -- with undefined upb_* symbols. + ["protoc"] = { + kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" }, + }, + }, -- protobuf's public headers #include "absl/…" directly, so Abseil is -- part of this package's interface, not an implementation detail. deps = { ["compat.abseil"] = "20250512.1" }, features = { + -- #355: libprotoc — the protobuf COMPILER library, which the `protoc` + -- target links. 138 TUs, transcribed from upstream's own + -- `src/file_lists.cmake` `libprotoc_srcs` (not hand-picked), and with + -- ZERO overlap against the runtime source set above: importer.cc and + -- parser.cc are already there. + -- + -- Off by default, and that is the whole point — a consumer that only + -- links the protobuf runtime must not compile these. + ["protoc"] = { + sources = { + "*/src/google/protobuf/compiler/code_generator.cc", + "*/src/google/protobuf/compiler/code_generator_lite.cc", + "*/src/google/protobuf/compiler/command_line_interface.cc", + "*/src/google/protobuf/compiler/cpp/enum.cc", + "*/src/google/protobuf/compiler/cpp/extension.cc", + "*/src/google/protobuf/compiler/cpp/field.cc", + "*/src/google/protobuf/compiler/cpp/field_chunk.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/map_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/message_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/string_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc", + "*/src/google/protobuf/compiler/cpp/file.cc", + "*/src/google/protobuf/compiler/cpp/generator.cc", + "*/src/google/protobuf/compiler/cpp/helpers.cc", + "*/src/google/protobuf/compiler/cpp/ifndef_guard.cc", + "*/src/google/protobuf/compiler/cpp/message.cc", + "*/src/google/protobuf/compiler/cpp/message_layout_helper.cc", + "*/src/google/protobuf/compiler/cpp/namespace_printer.cc", + "*/src/google/protobuf/compiler/cpp/parse_function_generator.cc", + "*/src/google/protobuf/compiler/cpp/service.cc", + "*/src/google/protobuf/compiler/cpp/tracker.cc", + "*/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc", + "*/src/google/protobuf/compiler/csharp/csharp_enum.cc", + "*/src/google/protobuf/compiler/csharp/csharp_enum_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_field_base.cc", + "*/src/google/protobuf/compiler/csharp/csharp_generator.cc", + "*/src/google/protobuf/compiler/csharp/csharp_helpers.cc", + "*/src/google/protobuf/compiler/csharp/csharp_map_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_message.cc", + "*/src/google/protobuf/compiler/csharp/csharp_message_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_primitive_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_message_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc", + "*/src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc", + "*/src/google/protobuf/compiler/csharp/names.cc", + "*/src/google/protobuf/compiler/java/context.cc", + "*/src/google/protobuf/compiler/java/doc_comment.cc", + "*/src/google/protobuf/compiler/java/field_common.cc", + "*/src/google/protobuf/compiler/java/file.cc", + "*/src/google/protobuf/compiler/java/full/enum.cc", + "*/src/google/protobuf/compiler/java/full/enum_field.cc", + "*/src/google/protobuf/compiler/java/full/extension.cc", + "*/src/google/protobuf/compiler/java/full/generator_factory.cc", + "*/src/google/protobuf/compiler/java/full/make_field_gens.cc", + "*/src/google/protobuf/compiler/java/full/map_field.cc", + "*/src/google/protobuf/compiler/java/full/message.cc", + "*/src/google/protobuf/compiler/java/full/message_builder.cc", + "*/src/google/protobuf/compiler/java/full/message_field.cc", + "*/src/google/protobuf/compiler/java/full/primitive_field.cc", + "*/src/google/protobuf/compiler/java/full/service.cc", + "*/src/google/protobuf/compiler/java/full/string_field.cc", + "*/src/google/protobuf/compiler/java/generator.cc", + "*/src/google/protobuf/compiler/java/helpers.cc", + "*/src/google/protobuf/compiler/java/internal_helpers.cc", + "*/src/google/protobuf/compiler/java/java_features.pb.cc", + "*/src/google/protobuf/compiler/java/lite/enum.cc", + "*/src/google/protobuf/compiler/java/lite/enum_field.cc", + "*/src/google/protobuf/compiler/java/lite/extension.cc", + "*/src/google/protobuf/compiler/java/lite/generator_factory.cc", + "*/src/google/protobuf/compiler/java/lite/make_field_gens.cc", + "*/src/google/protobuf/compiler/java/lite/map_field.cc", + "*/src/google/protobuf/compiler/java/lite/message.cc", + "*/src/google/protobuf/compiler/java/lite/message_builder.cc", + "*/src/google/protobuf/compiler/java/lite/message_field.cc", + "*/src/google/protobuf/compiler/java/lite/primitive_field.cc", + "*/src/google/protobuf/compiler/java/lite/string_field.cc", + "*/src/google/protobuf/compiler/java/message_serialization.cc", + "*/src/google/protobuf/compiler/java/name_resolver.cc", + "*/src/google/protobuf/compiler/java/names.cc", + "*/src/google/protobuf/compiler/java/shared_code_generator.cc", + "*/src/google/protobuf/compiler/kotlin/field.cc", + "*/src/google/protobuf/compiler/kotlin/file.cc", + "*/src/google/protobuf/compiler/kotlin/generator.cc", + "*/src/google/protobuf/compiler/kotlin/message.cc", + "*/src/google/protobuf/compiler/objectivec/enum.cc", + "*/src/google/protobuf/compiler/objectivec/enum_field.cc", + "*/src/google/protobuf/compiler/objectivec/extension.cc", + "*/src/google/protobuf/compiler/objectivec/field.cc", + "*/src/google/protobuf/compiler/objectivec/file.cc", + "*/src/google/protobuf/compiler/objectivec/generator.cc", + "*/src/google/protobuf/compiler/objectivec/helpers.cc", + "*/src/google/protobuf/compiler/objectivec/import_writer.cc", + "*/src/google/protobuf/compiler/objectivec/line_consumer.cc", + "*/src/google/protobuf/compiler/objectivec/map_field.cc", + "*/src/google/protobuf/compiler/objectivec/message.cc", + "*/src/google/protobuf/compiler/objectivec/message_field.cc", + "*/src/google/protobuf/compiler/objectivec/names.cc", + "*/src/google/protobuf/compiler/objectivec/oneof.cc", + "*/src/google/protobuf/compiler/objectivec/primitive_field.cc", + "*/src/google/protobuf/compiler/objectivec/tf_decode_data.cc", + "*/src/google/protobuf/compiler/php/names.cc", + "*/src/google/protobuf/compiler/php/php_generator.cc", + "*/src/google/protobuf/compiler/plugin.cc", + "*/src/google/protobuf/compiler/plugin.pb.cc", + "*/src/google/protobuf/compiler/python/generator.cc", + "*/src/google/protobuf/compiler/python/helpers.cc", + "*/src/google/protobuf/compiler/python/pyi_generator.cc", + "*/src/google/protobuf/compiler/retention.cc", + "*/src/google/protobuf/compiler/ruby/rbs_generator.cc", + "*/src/google/protobuf/compiler/ruby/ruby_generator.cc", + "*/src/google/protobuf/compiler/rust/accessors/accessor_case.cc", + "*/src/google/protobuf/compiler/rust/accessors/accessors.cc", + "*/src/google/protobuf/compiler/rust/accessors/default_value.cc", + "*/src/google/protobuf/compiler/rust/accessors/map.cc", + "*/src/google/protobuf/compiler/rust/accessors/repeated_field.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_cord.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_message.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_string.cc", + "*/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc", + "*/src/google/protobuf/compiler/rust/accessors/with_presence.cc", + "*/src/google/protobuf/compiler/rust/context.cc", + "*/src/google/protobuf/compiler/rust/crate_mapping.cc", + "*/src/google/protobuf/compiler/rust/enum.cc", + "*/src/google/protobuf/compiler/rust/extension.cc", + "*/src/google/protobuf/compiler/rust/generator.cc", + "*/src/google/protobuf/compiler/rust/message.cc", + "*/src/google/protobuf/compiler/rust/naming.cc", + "*/src/google/protobuf/compiler/rust/oneof.cc", + "*/src/google/protobuf/compiler/rust/relative_path.cc", + "*/src/google/protobuf/compiler/rust/rust_field_type.cc", + "*/src/google/protobuf/compiler/rust/rust_keywords.cc", + "*/src/google/protobuf/compiler/rust/upb_helpers.cc", + "*/src/google/protobuf/compiler/subprocess.cc", + "*/src/google/protobuf/compiler/versions.cc", + "*/src/google/protobuf/compiler/zip_writer.cc", + "*/upb_generator/common.cc", + "*/upb_generator/common/names.cc", + "*/upb_generator/file_layout.cc", + "*/upb_generator/minitable/names.cc", + "*/upb_generator/minitable/names_internal.cc", + "*/upb_generator/plugin.cc", + }, + }, -- GzipInputStream / GzipOutputStream. io/gzip_stream.cc is wrapped -- head-to-toe in `#if HAVE_ZLIB`, so by default it compiles to an -- empty TU and the package carries no zlib dependency at all; diff --git a/tests/examples/protobuf-protoc/build.mcpp b/tests/examples/protobuf-protoc/build.mcpp new file mode 100644 index 00000000..e1f95a5c --- /dev/null +++ b/tests/examples/protobuf-protoc/build.mcpp @@ -0,0 +1,69 @@ +// Generate inventory.pb.{h,cc} with the protoc this build produced. +// +// The work is DECLARED, not done here: `mcpp::action` makes it an edge in the +// build graph, so it re-runs exactly when the .proto changes and a failure is +// attributed to the edge rather than to "build.mcpp exited 1". +#include +#include + +import mcpp; + +namespace fs = std::filesystem; + +// protoc does NOT embed the well-known types. `import +// "google/protobuf/timestamp.proto"` is read from disk like any other import, +// and the files ship inside the protobuf package this project already depends +// on. Probe for the directory that actually contains descriptor.proto instead +// of hardcoding the tarball's wrap-directory name, which is a packaging +// artifact and not part of any contract. +static std::string well_known_types_dir() { + const std::string base = mcpp::dep_dir("protobuf"); + if (base.empty()) return {}; + std::error_code ec; + for (const auto& entry : fs::directory_iterator(base, ec)) { + const fs::path src = entry.path() / "src"; + if (fs::exists(src / "google" / "protobuf" / "descriptor.proto", ec)) + return src.generic_string(); + } + return {}; +} + +int main() { + const std::string root = mcpp::manifest_dir(); + const std::string out = mcpp::out_dir(); + + const char* protoc = mcpp::dep_bin("protobuf", "protoc"); + if (!protoc || !*protoc) { + std::fputs("no protoc: declare protobuf = { version = \"35.1\", " + "tools = [\"protoc\"] }\n", stderr); + return 1; + } + + const std::string wkt = well_known_types_dir(); + if (wkt.empty()) { + std::fputs("cannot locate the well-known .proto files in the protobuf " + "package\n", stderr); + return 1; + } + + const std::string proto = root + "/proto/inventory.proto"; + + // The .pb.h is declared alongside the .pb.cc because the test includes it + // and it must therefore be PRODUCED by this edge. mcpp knows a header is + // not a translation unit and keeps it out of the compile set. + mcpp::action gen; + gen.id = "protoc:inventory"; + gen.role = "source"; + gen.description = "protoc -> inventory"; + gen.arg(protoc) + .arg(("-I" + root + "/proto").c_str()) + .arg(("-I" + wkt).c_str()) + .arg(("--cpp_out=" + out).c_str()) + .arg(proto.c_str()) + .input(proto.c_str()) + .output((out + "/inventory.pb.cc").c_str()) + .output((out + "/inventory.pb.h").c_str()) + .submit(); + + mcpp::include_dir(out.c_str()); +} diff --git a/tests/examples/protobuf-protoc/mcpp.toml b/tests/examples/protobuf-protoc/mcpp.toml new file mode 100644 index 00000000..1589891d --- /dev/null +++ b/tests/examples/protobuf-protoc/mcpp.toml @@ -0,0 +1,23 @@ +# protobuf `protoc` target member — the compiler, not the runtime. +# +# The sibling tests/examples/protobuf covers the runtime and deliberately uses +# NO generated code. This member is its complement: every line of the message +# API it touches was emitted, during this build, by a protoc that mcpp built +# from the SAME descriptor that provides the runtime being linked. +# +# That co-provenance is the point. protoc and libprotobuf must agree on the +# generated-code ABI, and a mismatch there is a runtime failure, not a build +# error. Here it is not expressible: `tools = ["protoc"]` makes the tool's +# version the dependency's version. +[package] +name = "protobuf-protoc-tests" +version = "0.1.0" +standard = "c++23" + +[dependencies.compat] +# One dependency, two roles: `features` shapes what gets LINKED (the runtime), +# `tools` asks for a host binary out of the same package. `protoc` pulls in +# libprotoc's 138 TUs and needs `upb` for the upb generator's runtime — the +# descriptor's `required_features` states that, so asking for the tool is +# enough and this manifest does not have to know it. +protobuf = { version = "35.1", tools = ["protoc"] } diff --git a/tests/examples/protobuf-protoc/proto/inventory.proto b/tests/examples/protobuf-protoc/proto/inventory.proto new file mode 100644 index 00000000..edd57967 --- /dev/null +++ b/tests/examples/protobuf-protoc/proto/inventory.proto @@ -0,0 +1,37 @@ +// Small on purpose, but it exercises the generator features that break first +// when protoc and the linked runtime disagree: nested messages, an enum, a +// repeated message field, a map, oneof, and a well-known-type import. +syntax = "proto3"; + +package inventory; + +import "google/protobuf/timestamp.proto"; + +enum Grade { + GRADE_UNKNOWN = 0; + GRADE_A = 1; + GRADE_B = 2; +} + +message Item { + string sku = 1; + int32 quantity = 2; + Grade grade = 3; + + message Dimensions { + double width = 1; + double height = 2; + } + Dimensions dimensions = 4; + + oneof source { + string supplier = 5; + string warehouse = 6; + } +} + +message Inventory { + repeated Item items = 1; + map totals_by_grade = 2; + google.protobuf.Timestamp updated_at = 3; +} diff --git a/tests/examples/protobuf-protoc/tests/codegen.cpp b/tests/examples/protobuf-protoc/tests/codegen.cpp new file mode 100644 index 00000000..0f2726ee --- /dev/null +++ b/tests/examples/protobuf-protoc/tests/codegen.cpp @@ -0,0 +1,108 @@ +// Behavioral test for compat.protobuf's `protoc` target. +// +// Every type used here was emitted DURING THIS BUILD by a protoc that mcpp +// compiled from the same package that provides the runtime being linked. The +// test therefore asserts the thing that actually matters about a code +// generator shipped as a dependency: that its output and the runtime agree. +// +// What it drives, and what would break first on a generator/runtime mismatch: +// +// nested message + accessors generated_message_reflection.cc +// enum generated_enum_util.cc +// repeated message field repeated_ptr_field.cc +// map field map_field.cc +// oneof the generated case() discriminator +// well-known type import timestamp.pb.cc (proves the -I resolved) +// serialize -> parse round trip wire_format_lite.cc, parse_context.cc +// reflection over generated msg descriptor.cc against the generated pool +// +// Returns non-zero on any mismatch. +#include +#include + +#include "google/protobuf/util/time_util.h" + +#include "inventory.pb.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char* what) { + if (!ok) { + std::fprintf(stderr, "FAIL: %s\n", what); + ++failures; + } +} + +} // namespace + +int main() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + inventory::Inventory inv; + + inventory::Item* widget = inv.add_items(); + widget->set_sku("WIDGET-1"); + widget->set_quantity(7); + widget->set_grade(inventory::GRADE_A); + widget->mutable_dimensions()->set_width(2.5); + widget->mutable_dimensions()->set_height(4.0); + widget->set_supplier("acme"); + + inventory::Item* gizmo = inv.add_items(); + gizmo->set_sku("GIZMO-2"); + gizmo->set_quantity(3); + gizmo->set_grade(inventory::GRADE_B); + gizmo->set_warehouse("east"); + + (*inv.mutable_totals_by_grade())["A"] = 7; + (*inv.mutable_totals_by_grade())["B"] = 3; + + // The well-known type. Reaching this line at all proves protoc resolved + // the import, and setting it proves timestamp.pb.cc is in the runtime. + *inv.mutable_updated_at() = + google::protobuf::util::TimeUtil::SecondsToTimestamp(1735689600); + + std::string wire; + check(inv.SerializeToString(&wire), "serialize"); + check(!wire.empty(), "wire is non-empty"); + + inventory::Inventory back; + check(back.ParseFromString(wire), "parse"); + + check(back.items_size() == 2, "two items survived the round trip"); + check(back.items(0).sku() == "WIDGET-1", "item 0 sku"); + check(back.items(0).quantity() == 7, "item 0 quantity"); + check(back.items(0).grade() == inventory::GRADE_A, "item 0 enum"); + check(back.items(0).dimensions().width() == 2.5, "nested message field"); + check(back.items(0).source_case() == inventory::Item::kSupplier, + "oneof discriminator (supplier)"); + check(back.items(0).supplier() == "acme", "oneof value"); + check(back.items(1).source_case() == inventory::Item::kWarehouse, + "oneof discriminator (warehouse)"); + check(back.totals_by_grade().size() == 2, "map size"); + check(back.totals_by_grade().at("A") == 7, "map lookup"); + check(google::protobuf::util::TimeUtil::TimestampToSeconds( + back.updated_at()) == 1735689600, + "well-known Timestamp round trip"); + + // Reflection over the generated pool: the descriptor protoc emitted has to + // describe the C++ class it emitted beside it. + const google::protobuf::Descriptor* d = inventory::Item::descriptor(); + check(d != nullptr && d->full_name() == "inventory.Item", + "descriptor full name"); + check(d != nullptr && d->FindFieldByName("sku") != nullptr, + "descriptor knows the sku field"); + check(d != nullptr && d->oneof_decl_count() == 1, + "descriptor knows the oneof"); + + google::protobuf::ShutdownProtobufLibrary(); + + if (failures != 0) { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + std::puts("protoc-generated code round-trips against the linked runtime"); + return 0; +} From e17da76af1e92e4e3e9aedf4929d30e28e0678b2 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Wed, 5 Aug 2026 19:42:45 +0800 Subject: [PATCH 02/10] =?UTF-8?q?fix(protobuf):=20protoc=20=E7=9B=AE?= =?UTF-8?q?=E6=A0=87=E4=B8=8D=E5=A3=B0=E6=98=8E=E5=9C=A8=20windows;?= =?UTF-8?q?=E5=B9=B6=E8=AE=A9=E6=9C=AC=20PR=20=E4=B8=8D=E5=86=8D=E8=A7=A6?= =?UTF-8?q?=E5=8F=91=E5=85=A8=E9=87=8F=20workspace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI 暴露了两件独立的事。 ── 1. windows:工具子构建失败,所以不在那里声明这个目标 ────────────────── `workspace (windows)` 里 59 个成员挂了 3 个,其中一个是本 PR 新加的 protobuf-protoc。**不是 protobuf 的问题,也不是 flags 的问题** —— 同一次运行里 tests/examples/protobuf、protobuf-upb、protobuf-gzip 全部通过,用的是同一份 abseil + protobuf 源码,只不过是作为普通依赖构建的。只有**工具子构建**会死: error: building host tool 'compat.protobuf:protoc' failed error: cannot read 'obj/compat_abseil/…/absl/time/internal/test_util.cc.ddi' …/cctz/src/time_zone_posix.cc.ddi、…/cctz/src/zone_info_source.cc.ddi **不是路径长度。** MAX_PATH 是最顺手的猜测,而且是错的:这三个相对路径分别是 31 / 46 / 47 字符,而同一个子构建里 `absl/container/internal/hashtablez_sampler_force_weak_definition.cc`(67 字符) 编得好好的。子构建内层 ninja 的输出是被汇总过的,真正的 scan 报错没进日志, **原因未知**,也没有 windows 机器可复现。 在一个建不出来的平台上声明这个目标,等于把一个没有解释的失败甩给用户。所以 windows 平台块里放一份只含 `protobuf` 的 `targets`(平台块的 targets 覆盖顶层, compat.vulkan 早就在用这个机制),等有 windows 环境查清楚再放开。 成员也随之按 `[target.'cfg(os)']` 分平台:linux/macOS 要 `tools = ["protoc"]`, windows 只要运行时;build.mcpp 在 windows 上直接返回,tests/codegen.cpp 编成一条 **显式 skip** —— 一个什么都没验证却绿着的测试比红的更糟。 ── 2. 还原 index.toml:它让 CI 跑全量 workspace 并超时 ──────────────────── 成员选择把 `index.toml` 归到「未分类改动」→ `MEMBERS=__ALL__` → 59 个成员全建。 linux leg 因此**恰好跑满 150 分钟 timeout 被取消**(09:04:17 → 11:34:34), 其间每一条测试都是通过的。macOS 83 分钟侥幸跑完。 而我改它只是把 `latest_mcpp` 从 2026.8.3.3 抬到 2026.8.5.1 —— 一个**引擎里零消费者** 的提示字段(已 grep 确认:只解析、从不使用)。为一个纯装饰的字段换来 2.5 小时全量 构建并超时,不值。还原之后选择器只挑 protobuf 相关成员。 `min_mcpp` 本来就不动,那个判断没变。 --- index.toml | 12 +------- pkgs/c/compat.protobuf.lua | 29 +++++++++++++++++++ tests/examples/protobuf-protoc/build.mcpp | 5 ++++ tests/examples/protobuf-protoc/mcpp.toml | 16 +++++++++- .../protobuf-protoc/tests/codegen.cpp | 12 ++++++++ 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/index.toml b/index.toml index 8e64e621..2793f7ff 100644 --- a/index.toml +++ b/index.toml @@ -7,15 +7,5 @@ # "floor first, new grammar after" rollout rule mechanically. [index] spec = "1" -# -# These two move independently, and the 2026.8.5.1 CI pin is why. `min_mcpp` is -# a GATE — an older client that fails it cannot open this index at all -# (mcpp#349), so it moves only when a descriptor genuinely stops being readable -# by the older grammar. Exposing compat.protobuf's `protoc` target did not do -# that: 2026.8.3.3 still parses the descriptor with an empty `unknown_keys`. -# What needs 2026.8.5.1 is the CONSUMER spelling `tools = [...]`, which lives -# in a consumer's own mcpp.toml — including this repo's -# tests/examples/protobuf-protoc, hence the CI pin. `latest_mcpp` is an -# advisory hint with no gate behaviour, and tracks what CI validates against. min_mcpp = "2026.8.3.3" -latest_mcpp = "2026.8.5.1" +latest_mcpp = "2026.8.3.3" diff --git a/pkgs/c/compat.protobuf.lua b/pkgs/c/compat.protobuf.lua index 4ba1d892..7d7c7b01 100644 --- a/pkgs/c/compat.protobuf.lua +++ b/pkgs/c/compat.protobuf.lua @@ -501,6 +501,35 @@ package = { -- here each package carries its own compile flags, so it has to be -- stated. No extra import libs: -ladvapi32 arrives with abseil. cxxflags = { "-DNOMINMAX", "-DWIN32_LEAN_AND_MEAN", "-D_CRT_SECURE_NO_WARNINGS" }, + + -- NO `protoc` TARGET ON WINDOWS — a platform `targets` replaces the + -- top-level one, so this drops the tool while keeping the library. + -- + -- Not a protobuf problem and not a flags problem: the tool SUB-BUILD + -- fails there. In the same CI run, tests/examples/protobuf, + -- protobuf-upb and protobuf-gzip all pass on windows — the very same + -- abseil + protobuf sources, built as an ordinary dependency. Only + -- the sub-build dies, and only on three abseil TUs whose `.ddi` scan + -- outputs never appear: + -- + -- error: building host tool 'compat.protobuf:protoc' failed + -- error: cannot read 'obj/compat_abseil/…/absl/time/internal/test_util.cc.ddi' + -- …/cctz/src/time_zone_posix.cc.ddi, …/cctz/src/zone_info_source.cc.ddi + -- + -- It is NOT path length (MAX_PATH was the obvious guess and it is + -- wrong: those three relative paths are 31/46/47 chars, while + -- absl/container/internal/hashtablez_sampler_force_weak_definition.cc + -- at 67 compiles fine in the same sub-build). The sub-build's inner + -- ninja output is summarized, so the underlying scan error is not in + -- the log and the cause is UNKNOWN. + -- + -- Declaring the target on a platform where it cannot be built would + -- hand users a failure with no explanation. Left off until the + -- sub-build issue is diagnosed on a windows host; nothing else about + -- this descriptor is windows-gated. + targets = { + ["protobuf"] = { kind = "lib" }, + }, }, }, } diff --git a/tests/examples/protobuf-protoc/build.mcpp b/tests/examples/protobuf-protoc/build.mcpp index e1f95a5c..9a0458e7 100644 --- a/tests/examples/protobuf-protoc/build.mcpp +++ b/tests/examples/protobuf-protoc/build.mcpp @@ -29,6 +29,11 @@ static std::string well_known_types_dir() { } int main() { + // No `protoc` target on windows (see the descriptor's windows block), so + // there is nothing to declare and no include dir to add. Returning 0 keeps + // the member building; tests/codegen.cpp compiles to a visible skip. + if (std::string(mcpp::target_os()) == "windows") return 0; + const std::string root = mcpp::manifest_dir(); const std::string out = mcpp::out_dir(); diff --git a/tests/examples/protobuf-protoc/mcpp.toml b/tests/examples/protobuf-protoc/mcpp.toml index 1589891d..82ae3b1d 100644 --- a/tests/examples/protobuf-protoc/mcpp.toml +++ b/tests/examples/protobuf-protoc/mcpp.toml @@ -9,15 +9,29 @@ # generated-code ABI, and a mismatch there is a runtime failure, not a build # error. Here it is not expressible: `tools = ["protoc"]` makes the tool's # version the dependency's version. +# +# WINDOWS: linux + macOS only, matching the descriptor — compat.protobuf does +# not declare the `protoc` target on windows, because the tool sub-build fails +# there for reasons not yet diagnosed (see the comment in the descriptor's +# windows block). The dependency below is therefore per-OS, and tests/codegen.cpp +# compiles to a visible skip on windows rather than a test that silently proves +# nothing. [package] name = "protobuf-protoc-tests" version = "0.1.0" standard = "c++23" -[dependencies.compat] # One dependency, two roles: `features` shapes what gets LINKED (the runtime), # `tools` asks for a host binary out of the same package. `protoc` pulls in # libprotoc's 138 TUs and needs `upb` for the upb generator's runtime — the # descriptor's `required_features` states that, so asking for the tool is # enough and this manifest does not have to know it. +[target.'cfg(linux)'.dependencies.compat] +protobuf = { version = "35.1", tools = ["protoc"] } + +[target.'cfg(macos)'.dependencies.compat] protobuf = { version = "35.1", tools = ["protoc"] } + +# The runtime alone, so the member still builds and links something real here. +[target.'cfg(windows)'.dependencies.compat] +protobuf = "35.1" diff --git a/tests/examples/protobuf-protoc/tests/codegen.cpp b/tests/examples/protobuf-protoc/tests/codegen.cpp index 0f2726ee..d70c0f71 100644 --- a/tests/examples/protobuf-protoc/tests/codegen.cpp +++ b/tests/examples/protobuf-protoc/tests/codegen.cpp @@ -20,6 +20,16 @@ #include #include +// compat.protobuf declares no `protoc` target on windows, so nothing was +// generated and there is nothing to assert. A loud skip beats a test that +// passes without exercising anything. +#ifdef _WIN32 +int main() { + std::puts("skipped: compat.protobuf has no protoc target on windows"); + return 0; +} +#else + #include "google/protobuf/util/time_util.h" #include "inventory.pb.h" @@ -106,3 +116,5 @@ int main() { std::puts("protoc-generated code round-trips against the linked runtime"); return 0; } + +#endif // _WIN32 From d2efefa1c26c2ace4d75598555184530b3a36fce Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Wed, 5 Aug 2026 22:17:38 +0800 Subject: [PATCH 03/10] =?UTF-8?q?ci:=20=E6=89=93=E5=BC=80=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E5=8C=85=E7=BC=93=E5=AD=98,pin=20=E5=88=B0=202026.8.5?= =?UTF-8?q?.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 全量运行(改 validate.yml 会强制全量,这是刻意的)跑不完: linux 2h30m22s -> 跑满 150 分钟 timeout 被取消 windows 2h20m08s -> opencv-module / opencv-module-dnn 链接失败 macos 1h26m45s -> 通过 两个原因,各有各的修法。 ── 1. 每个 member 都从零重编所有依赖 ──────────────────────────────────────── 这一步一直设着 `MCPP_BUILD_CACHE: local`,把**全局包构建缓存**关掉了。它是 mcpp#344 的临时规避:对象路径消歧按「整个 build dir 的 basename 冲突」触发, 即取决于**消费方**拉了哪些包,而当时的缓存键只覆盖依赖自身,于是一个条目能装下 两种布局,ninja 在图加载阶段就死。 **#344 已在 2026.8.3.4 修掉**(per-package Merkle 键,覆盖随消费方变化的布局), 理由不存在了。而留着它是有代价的:59 个 member 大量共享 abseil / protobuf / opencv,关掉缓存意味着同一份源码被反复编译 —— opencv 系每个 10~20 分钟。 打开之后,同一个 (包, 版本, features, 工具链) 每次运行只建一次,后面的 member 直接命中。 > 注:CI 的 actions/cache 只缓存 `~/.mcpp/registry`(工具链与包源码),不缓存 > `~/.mcpp/build-cache`。所以本次拿到的是**同一次运行内**的复用 —— 而那正是 > 超时的来源。跨运行缓存是另一件事:GitHub 每仓库 10GB 上限,opencv 的产物有 > 撑爆并引发频繁驱逐的风险,先用数据说话再决定。 ── 2. windows 上 opencv 链接失败 ─────────────────────────────────────────── fatal error LNK1170: line in command file contains 135135 or more characters link.exe 的响应文件**单行**上限 128 KiB,而 mcpp 此前把所有对象写在一行。 mcpp 2026.8.5.3 改成按行分隔(`$in_newline`),上限不再随对象数增长。 编译完 795s / 1166s 之后才在最后一步倒下,这个代价尤其刺眼。 因此 pin 抬到 **2026.8.5.3**。`index.toml` 的 `min_mcpp` 不动 —— 这两条都不改变 任何描述符是否可被旧客户端读取。 --- .github/workflows/validate.yml | 73 +++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b292c934..707f4427 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -14,11 +14,21 @@ on: workflow_dispatch: env: - # 2026.8.5.1 is a FLOOR, not a routine bump: `tools = [...]` — how a consumer - # asks for a dependency's `kind = "bin"` target, which the new - # tests/examples/protobuf-protoc member is built on — does not parse before - # it ("tools must be a string, inline dep table, or nested table"). Note this - # does NOT move index.toml's min_mcpp: exposing compat.protobuf's `protoc` + # 2026.8.5.3 carries two things this workflow depends on: + # .5.1 `tools = [...]` — how a consumer asks for a dependency's + # `kind = "bin"` target, which tests/examples/protobuf-protoc is built + # on. Before it: "tools must be a string, inline dep table, or nested + # table". + # .5.3 link response files separate objects by NEWLINES. link.exe caps a + # response-file LINE at 128 KiB, and opencv-module / opencv-module-dnn + # went past it on windows — after 795s / 1166s of compiling: + # fatal error LNK1170: line in command file contains 135135 or + # more characters + # Together with re-enabling the global package cache below, this is + # what makes a green FULL run possible again: .5.3 removes the + # windows link failure, the cache removes the 150-minute timeout. + # + # Neither of them moves index.toml's min_mcpp: exposing compat.protobuf's `protoc` # target is additive, and 2026.8.3.3 still parses that descriptor with an # empty unknown_keys. The floor an index publishes decides whether older # clients keep working at all (mcpp#349), so it moves only when a descriptor @@ -97,7 +107,7 @@ env: # 0.0.94 fixed feature-gated `sources` under `mcpp test` (mcpp#218); 0.0.91 # added standard = "c++fly" to the resolver grammar, so c++fly descriptors # get the lint WARN below, not a hard grammar-parse rejection. - MCPP_VERSION: "2026.8.5.1" + MCPP_VERSION: "2026.8.5.3" jobs: lint: @@ -262,21 +272,21 @@ jobs: ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.5.1" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.3" # keep in sync with env.MCPP_VERSION - platform: macos os: macos-15 suffix: macosx-arm64 ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.5.1" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.3" # keep in sync with env.MCPP_VERSION - platform: windows os: windows-latest suffix: windows-x86_64 ext: zip mcpp: bin/mcpp.exe xlings: registry/bin/xlings.exe - mcpp_version: "2026.8.5.1" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.3" # keep in sync with env.MCPP_VERSION env: MCPP_EFFECTIVE: ${{ matrix.mcpp_version }} steps: @@ -446,29 +456,30 @@ jobs: shell: bash env: MCPP_INDEX_MIRROR: GLOBAL - # Dependencies build inside each member's own target/ instead of - # through the global package build cache (mcpp >= 2026.7.30.2). - # That cache is unusable here: mcpp#233's object-path disambiguation - # fires on basename collisions across the WHOLE build dir — i.e. on - # which packages the CONSUMER pulls in — while the cache key covers - # only the dependency itself, so one entry can hold two different - # layouts. `tests/examples/archive` pulls zlib AND bzip2 (both ship - # compress.c) and stores obj/compat_zlib/zlib-1.3.2/compress.o; - # every zlib consumer without bzip2 then asks the same key for a - # flat obj/compress.o and ninja dies at graph time with - # "missing and no known rule to make it". Reproduced both ways round - # on 2026.8.3.3 and filed as mcpp-community/mcpp#344. `local` still - # caches the std BMI, which is the expensive one — only package - # entries are bypassed. + # The GLOBAL package build cache is on (mcpp >= 2026.7.30.2), which + # is the default — this step used to set `MCPP_BUILD_CACHE: local` + # and no longer does. + # + # That bypass existed for mcpp#344: object-path disambiguation fires + # on basename collisions across the WHOLE build dir — i.e. on what + # the CONSUMER pulls in — while the cache key covered only the + # dependency, so one entry could hold two layouts and ninja died at + # graph time with "missing and no known rule to make it". #344 + # landed in 2026.8.3.4 with per-package Merkle keys that cover the + # consumer-dependent layout, so the reason is gone. + # + # Keeping it cost real time, and the full run is where it showed: + # with `local`, EVERY member recompiles EVERY dependency from + # scratch. 59 members that mostly share abseil / protobuf / opencv + # meant the same sources were built over and over — + # + # linux 2h30m -> cancelled at the 150-minute timeout + # windows 2h20m + # macos 1h26m # - # STATUS: #344 landed in 2026.8.3.4 (per-package Merkle keys that - # cover the consumer-dependent layout), so as of the pin above this - # bypass is no longer required. It is kept here deliberately: - # re-enabling the global package cache changes how EVERY member - # builds, and that deserves its own PR where a failure is - # unambiguous rather than being attributed to whatever else shipped - # alongside it. - MCPP_BUILD_CACHE: local + # — and a workspace cannot be validated by a job that cannot finish. + # With the cache on, a given (package, version, features, toolchain) + # is built once per run and every later member hits it. run: | "$MCPP" --version # No `timeout` wrapper: absent on macOS runners; job-level timeout-minutes bounds it. From 7218bef7ac14ec84b83f49b856945a8427821e35 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 00:07:39 +0800 Subject: [PATCH 04/10] =?UTF-8?q?ci:=20=E5=85=A8=E9=87=8F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E6=8C=89=20member=20=E5=88=86=208=20=E7=89=87?= =?UTF-8?q?=E5=B9=B6=E8=A1=8C,=E6=AF=8F=E5=B9=B3=E5=8F=B0=208=20=E4=B8=AA?= =?UTF-8?q?=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 全量运行的墙钟由**一个 job 串行跑完 59 个 member**决定,其中 opencv 系每个 15~20 分钟。linux 因此跑满 150 分钟 timeout 被取消 —— 而它每一条测试都是过的, 只是跑不完。一个验证不完的 workspace 等于没有验证。 改成按 member 轮转分片,每平台 8 个 runner 并行。 ── 只在全量时扇出 ───────────────────────────────────────────────────────── 成员选择原本内联在 workspace job 里,三个平台各自重算一遍同一个答案。现在提成 前置 job `select`,因为**矩阵的 shard 维度取决于它**: shards = [0..7] 全量 shards = [0] 选择性运行 矩阵用 `fromJSON(needs.select.outputs.shards)` 展开,所以选择性运行仍然是每平台 一个 job。给几个 member 分 8 片毫无意义 —— 只会多出 7 份 checkout、mcpp 下载和 缓存恢复。job 名字也只在分片时才带 `0/8`。 ── 轮转分片把重的拆开 ────────────────────────────────────────────────────── 按位置取模。opencv-module / -dnn / -unifont 在列表里相邻,`% 8` 必然把它们放到 **三个不同的 runner** 上,这正是要解决的那件事: shard 3: opencv-module shard 4: opencv-module-dnn + llamacpp shard 5: opencv-module-unifont + ffmpeg + llamacpp-metal **这个分配并不理想,说清楚**:取模不知道每个 member 要跑多久,均衡靠运气。 shard 5 拿了 3 个重量级,shard 0/1/2 一个都没有,而墙钟由最慢的那片决定。要真正 均衡就得维护一张耗时表 —— 本文件别处已经写明「不要再引入手维护的清单」,那种 清单会悄悄过期。先用无信息但零维护的分法,等实测数据说明它不够再谈。 timeout 从 150 降到 90:一片约是 1/8 的活,90 分钟是一道真正的上限,而不是决定 job 能不能跑完的那个数。 ── 与全局缓存互补 ───────────────────────────────────────────────────────── 两者管的是不同的重复:片内靠缓存复用同一个 (包,版本,features,工具链),片间靠 并行。跨 runner 不共享 build-cache,所以 abseil 这类共享依赖会在 8 个片里各编 一次 —— 但它们是并行的,墙钟只算一次。 --- .github/workflows/validate.yml | 210 ++++++++++++++++++++++----------- 1 file changed, 139 insertions(+), 71 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 707f4427..703905d7 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -255,13 +255,124 @@ jobs: # `dnn` feature members. The registry cache (restore-keys prefix below) # amortizes those across subsequent runs. 150 covers the one-time cold full # build with headroom; it is a ceiling, not a target. + # ── The plan, computed ONCE ─────────────────────────────────────────── + # Was inlined in every workspace job — three runners each re-deriving the + # same answer. It now also has to be decided BEFORE the matrix exists, + # because the matrix's shard dimension depends on it: a full run fans out, + # a selective one does not. + select: + runs-on: ubuntu-latest + outputs: + members: ${{ steps.fanout.outputs.members }} + shard_count: ${{ steps.fanout.outputs.shard_count }} + shards: ${{ steps.fanout.outputs.shards }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # ── Selective member testing ────────────────────────────────────── + # `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and + # dominates CI wall-clock, while a PR almost always touches one + # package. Map changed files → affected members and test only those: + # pkgs//.lua → members whose mcpp.toml references + # tests/examples//** → member + # Run the FULL workspace when the change can affect everything: + # non-PR events (push to main, the nightly cron, dispatch), this + # workflow file (it carries the mcpp version pins, so a version bump + # always re-validates every package), a non-member edit to the + # workspace manifest, or shared test scripts. Docs-only and tools/-only + # changes select nothing. + # Note: bash 3.2 on macOS runners — no associative arrays here. + - name: Select affected workspace members + id: plan + shell: bash + run: | + full() { echo "MEMBERS=__ALL__" >> "$GITHUB_ENV"; echo "full run: $1"; exit 0; } + [ "${{ github.event_name }}" = "pull_request" ] || full "event=${{ github.event_name }}" + base="origin/${{ github.base_ref }}" + changed=$(git diff --name-only "$base"...HEAD) + printf 'changed files vs %s:\n%s\n' "$base" "$changed" + sel="" + add() { case " $sel " in *" $1 "*) ;; *) sel="$sel $1" ;; esac; } + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + .github/workflows/validate.yml|tests/*.sh) full "$f" ;; + mcpp.toml) + # Workspace manifest. Every new-package PR appends to the + # members list, so that alone must NOT force a full run: + # select the added members; anything else in this file + # (indices, settings) affects everyone → full. + if ! diff -q <(git show "$base:mcpp.toml" | grep -v 'tests/examples/') \ + <(grep -v 'tests/examples/' mcpp.toml) >/dev/null; then + full "mcpp.toml non-member change" + fi + for p in $(comm -13 <(git show "$base:mcpp.toml" | grep -o 'tests/examples/[A-Za-z0-9._-]*' | sort -u) \ + <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml | sort -u)); do + add "${p#tests/examples/}" + done ;; + tests/examples/*) + m=${f#tests/examples/}; m=${m%%/*} + # A deleted/renamed member dir implies a mcpp.toml edit, + # which already forces a full run above. + [ -d "tests/examples/$m" ] && add "$m" ;; + pkgs/*.lua|pkgs/*/*.lua) + lib=$(basename "$f" .lua); lib=${lib#compat.} + hit=0 + for mt in tests/examples/*/mcpp.toml; do + if grep -q "$lib" "$mt"; then add "$(basename "$(dirname "$mt")")"; hit=1; fi + done + [ "$hit" = 1 ] || echo "note: no workspace member exercises $f" ;; + # tools/ holds OFFLINE descriptor-generation and publishing + # helpers (tools/compat-*/, tools/gtc/, publish_mcpp_index.sh). + # Nothing under it is consumed by a package build: when one of + # them actually changes a package, the generated pkgs/*.lua + # changes with it and the rule above selects the right members. + # So a tools/ edit alone selects nothing rather than forcing a + # full workspace rebuild. + *.md|docs/*|.agents/*|.github/*|tools/*) : ;; + *) full "unclassified change: $f" ;; + esac + done <> "$GITHUB_ENV" + echo "selected members: ${sel:-}" + # Sharding is for the FULL run only. A selective run is a handful of + # members; splitting it 8 ways would add 7 runners' worth of checkout, + # mcpp download and cache restore to save nothing. + - name: Decide the fan-out + id: fanout + shell: bash + run: | + if [ "$MEMBERS" = "__ALL__" ]; then + echo 'shards=[0,1,2,3,4,5,6,7]' >> "$GITHUB_OUTPUT" + echo 'shard_count=8' >> "$GITHUB_OUTPUT" + else + echo 'shards=[0]' >> "$GITHUB_OUTPUT" + echo 'shard_count=1' >> "$GITHUB_OUTPUT" + fi + echo "members=$MEMBERS" >> "$GITHUB_OUTPUT" + workspace: - name: workspace (${{ matrix.platform }}) + # The shard is in the name only when there is more than one, so a + # selective run still reads "workspace (linux)". + name: workspace (${{ matrix.platform }}${{ needs.select.outputs.shard_count == '1' && '' || format(' {0}/{1}', matrix.shard, needs.select.outputs.shard_count) }}) + needs: select + if: needs.select.outputs.members != '' runs-on: ${{ matrix.os }} - timeout-minutes: 150 + # Was 150 and a full linux run hit it exactly. One shard is ~1/8 of the + # work, so this is now a real ceiling rather than the thing that decides + # whether the job finishes. + timeout-minutes: 90 strategy: fail-fast: false matrix: + # Fanned out ONLY for a full run — `shards` is [0] otherwise, which + # collapses this back to one job per platform. + shard: ${{ fromJSON(needs.select.outputs.shards) }} + platform: [linux, macos, windows] include: # Archive names are derived from env.MCPP_VERSION in the Download # step — bumping the pin is a ONE-line change (hardcoded versions @@ -359,74 +470,29 @@ jobs: # plans, mcpp#232). The sandbox copy lands in ~/.mcpp/registry, so # the cache carries it across runs. - # ── Selective member testing ────────────────────────────────────── - # `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and - # dominates CI wall-clock, while a PR almost always touches one - # package. Map changed files → affected members and test only those: - # pkgs//.lua → members whose mcpp.toml references - # tests/examples//** → member - # Run the FULL workspace when the change can affect everything: - # non-PR events (push to main, the nightly cron, dispatch), this - # workflow file (it carries the mcpp version pins, so a version bump - # always re-validates every package), a non-member edit to the - # workspace manifest, or shared test scripts. Docs-only and tools/-only - # changes select nothing. - # Note: bash 3.2 on macOS runners — no associative arrays here. - - name: Select affected workspace members + # ── This shard's slice of the plan ──────────────────────────────── + # `select` decided WHAT runs; this decides which part of it runs HERE. + # Round-robin by position, which is what spreads the expensive members: + # opencv-module / -dnn / -unifont are adjacent in the list, so `% N` + # necessarily puts them on three different runners. A single job that + # builds all three spends 45+ minutes on opencv alone. + - name: Take this shard's members shell: bash run: | - full() { echo "MEMBERS=__ALL__" >> "$GITHUB_ENV"; echo "full run: $1"; exit 0; } - [ "${{ github.event_name }}" = "pull_request" ] || full "event=${{ github.event_name }}" - base="origin/${{ github.base_ref }}" - changed=$(git diff --name-only "$base"...HEAD) - printf 'changed files vs %s:\n%s\n' "$base" "$changed" - sel="" - add() { case " $sel " in *" $1 "*) ;; *) sel="$sel $1" ;; esac; } - while IFS= read -r f; do - [ -n "$f" ] || continue - case "$f" in - .github/workflows/validate.yml|tests/*.sh) full "$f" ;; - mcpp.toml) - # Workspace manifest. Every new-package PR appends to the - # members list, so that alone must NOT force a full run: - # select the added members; anything else in this file - # (indices, settings) affects everyone → full. - if ! diff -q <(git show "$base:mcpp.toml" | grep -v 'tests/examples/') \ - <(grep -v 'tests/examples/' mcpp.toml) >/dev/null; then - full "mcpp.toml non-member change" - fi - for p in $(comm -13 <(git show "$base:mcpp.toml" | grep -o 'tests/examples/[A-Za-z0-9._-]*' | sort -u) \ - <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml | sort -u)); do - add "${p#tests/examples/}" - done ;; - tests/examples/*) - m=${f#tests/examples/}; m=${m%%/*} - # A deleted/renamed member dir implies a mcpp.toml edit, - # which already forces a full run above. - [ -d "tests/examples/$m" ] && add "$m" ;; - pkgs/*.lua|pkgs/*/*.lua) - lib=$(basename "$f" .lua); lib=${lib#compat.} - hit=0 - for mt in tests/examples/*/mcpp.toml; do - if grep -q "$lib" "$mt"; then add "$(basename "$(dirname "$mt")")"; hit=1; fi - done - [ "$hit" = 1 ] || echo "note: no workspace member exercises $f" ;; - # tools/ holds OFFLINE descriptor-generation and publishing - # helpers (tools/compat-*/, tools/gtc/, publish_mcpp_index.sh). - # Nothing under it is consumed by a package build: when one of - # them actually changes a package, the generated pkgs/*.lua - # changes with it and the rule above selects the right members. - # So a tools/ edit alone selects nothing rather than forcing a - # full workspace rebuild. - *.md|docs/*|.agents/*|.github/*|tools/*) : ;; - *) full "unclassified change: $f" ;; - esac - done <> "$GITHUB_ENV" - echo "selected members: ${sel:-}" + plan='${{ needs.select.outputs.members }}' + if [ "$plan" = "__ALL__" ]; then + plan=$(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml \ + | sed 's|tests/examples/||' | sort -u | tr '\n' ' ') + fi + n=${{ needs.select.outputs.shard_count }} + mine=""; i=0 + for m in $plan; do + [ $((i % n)) -eq ${{ matrix.shard }} ] && mine="$mine $m" + i=$((i + 1)) + done + mine=${mine# } + echo "MEMBERS=$mine" >> "$GITHUB_ENV" + echo "shard ${{ matrix.shard }}/$n of $i member(s): ${mine:-}" # ── Refresh the PUBLISHED index before testing ──────────────────── # Most members resolve everything from this checkout, but a member that @@ -483,9 +549,11 @@ jobs: run: | "$MCPP" --version # No `timeout` wrapper: absent on macOS runners; job-level timeout-minutes bounds it. - if [ "$MEMBERS" = "__ALL__" ]; then - "$MCPP" test --workspace - elif [ -z "$MEMBERS" ]; then + # One code path: the shard step above already expanded `__ALL__` + # into this runner's actual member names, so `mcpp test --workspace` + # — which would ignore the sharding and rebuild everything here — is + # gone. + if [ -z "$MEMBERS" ]; then echo "No workspace member affected by this change — nothing to test." else rc=0 From cf93e248285ef4e72f4ebc467b742be16d7d05b7 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 00:28:26 +0800 Subject: [PATCH 05/10] =?UTF-8?q?ci:=20=E6=89=8B=E5=8A=A8=E8=A7=A6?= =?UTF-8?q?=E5=8F=91=E5=8F=AF=E5=88=87=E6=8D=A2=E7=BC=93=E5=AD=98=20+=20?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E8=B7=91=E5=AE=8C=E7=BB=99=E5=87=BA=E6=AF=8F?= =?UTF-8?q?=E4=B8=AA=E6=88=90=E5=91=98=E7=9A=84=E8=80=97=E6=97=B6=E6=8E=92?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 三件事,都先在本地跑通再接进 CI。 ── 1. tests/run_members.sh —— 本地与 CI 用同一份 ──────────────────────────── 把内联在 workflow 里的循环抽成脚本。理由不是整洁:**只存在于 CI 的耗时表没法用来 决定优化什么**,而与 CI 不一致的本地脚手架量的是另一回事。 bash tests/run_members.sh --all bash tests/run_members.sh --all --shard 3/8 bash tests/run_members.sh --all --cache local bash tests/run_members.sh opencv-module protobuf 本地实测发现一个 CI 上才会炸的 bug:`--all` 从 mcpp.toml 里 grep 成员名,而该文件 第 3 行的**散文**也写了 `tests/examples/`(「tests/examples/ — each consumes...」), 去掉前缀后是**空字符串** —— CI 上会变成 `mcpp test -p ""`。现在同时过滤空名和 不存在的目录(注释里被讨论到的 `tests/examples/asio-ssl` 也因此不会变成幽灵成员)。 过滤后 59 个,与目录一致。 失败不吞:任一成员失败则退出码非零,但**耗时表照常打印** —— 跑挂了的那次恰恰最 需要知道时间花在哪。 ── 2. workflow_dispatch 可选缓存模式 ─────────────────────────────────────── cache: global(默认)| local `local` 让每个成员各自重编全部依赖,是耗时表的对照基准。非 dispatch 事件时该 input 为空字符串,而 mcpp 的 `resolve_cache_mode` 只在**非空**时才认这个环境变量, 所以天然回落到默认的 global,不需要额外分支。 ── 3. 耗时排名 ──────────────────────────────────────────────────────────── 分片把成本藏起来了:八个 runner 各报各的,没人看得见到底谁在吃时间。每片把 `<秒>\t<成员>\t` 传成 artifact,新增 `timings` job 按平台合并,排名写进 run summary(带占比)。`always()` —— 失败的那次正是最该读它的时候。 汇总的合并/排序/占比逻辑已在本地用构造数据验证过。 表里的 total 是**各片之和**,墙钟是**最慢那片** —— 这两个数不是一回事,summary 里写明了,免得下次拿总和当墙钟看。 --- .github/workflows/validate.yml | 83 ++++++++++++++++++++++--- tests/run_members.sh | 107 +++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 7 deletions(-) create mode 100755 tests/run_members.sh diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 703905d7..4a0c1745 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -12,6 +12,12 @@ on: # nightly full regression — exercises every workspace member regardless of diff - cron: "0 6 * * *" workflow_dispatch: + inputs: + cache: + description: "Package build cache — 'local' rebuilds every dependency per member, which is what the timing table should be read against when comparing" + type: choice + options: [global, local] + default: global env: # 2026.8.5.3 carries two things this workflow depends on: @@ -553,18 +559,27 @@ jobs: # into this runner's actual member names, so `mcpp test --workspace` # — which would ignore the sharding and rebuild everything here — is # gone. + # + # tests/run_members.sh is the SAME script you run locally. A timing + # table that only exists in CI cannot be used while deciding what to + # optimise, and a local harness that differs from CI measures + # something else. if [ -z "$MEMBERS" ]; then echo "No workspace member affected by this change — nothing to test." else - rc=0 - for m in $MEMBERS; do - echo "::group::mcpp test -p $m" - "$MCPP" test -p "$m" || rc=1 - echo "::endgroup::" - done - exit $rc + MCPP_TIMINGS="$PWD/timings.tsv" bash tests/run_members.sh $MEMBERS fi + # Per-shard timings, merged by the `timings` job below. `always()`: a + # run that failed is exactly when knowing where the time went matters. + - name: Upload this shard's timings + if: always() && hashFiles('timings.tsv') != '' + uses: actions/upload-artifact@v4 + with: + name: timings-${{ matrix.platform }}-${{ matrix.shard }} + path: timings.tsv + retention-days: 14 + # install()-driven packages (openssl, openblas) build through their own # Make/Configure system, whose output xim's interface mode swallows; a # failed hook surfaces only as `E_INTERNAL: [] failed:`. Each writes @@ -583,3 +598,57 @@ jobs: echo "::endgroup::" done < <(find tests/examples "$HOME/.mcpp/registry" -name 'mcpp_*_build.log' 2>/dev/null) [ "$found" = 1 ] || echo "no install() build logs found" + + # ── Where the time went ─────────────────────────────────────────────── + # Sharding hides the cost: eight runners each report their own slice, and + # nobody can see which members actually dominate. This merges them into one + # ranking per platform, in the run summary, so the next optimisation starts + # from measurement instead of a guess. + # + # `always()` — a failed run is exactly when this is worth reading. + timings: + needs: [select, workspace] + if: always() && needs.select.outputs.members != '' + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + pattern: timings-* + path: timings + continue-on-error: true + - name: Rank members by wall-clock + shell: bash + run: | + shopt -s nullglob + files=(timings/*/timings.tsv) + if [ ${#files[@]} -eq 0 ]; then + echo "no timing data (every shard skipped or failed before testing)" \ + >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Artifact name carries the platform: timings--. + for plat in linux macos windows; do + rows=$(mktemp) + for f in timings/timings-$plat-*/timings.tsv; do + [ -f "$f" ] && cat "$f" >> "$rows" + done + [ -s "$rows" ] || { rm -f "$rows"; continue; } + + total=$(awk -F'\t' '{s += $1} END {print s+0}' "$rows") + count=$(wc -l < "$rows") + { + echo "### $plat — ${count} member(s), ${total}s of member wall-clock" + echo + echo "| rank | seconds | share | member | result |" + echo "|---:|---:|---:|---|---|" + sort -rn "$rows" | awk -F'\t' -v tot="$total" ' + { pct = tot > 0 ? ($1 * 100 / tot) : 0 + printf "| %d | %s | %.1f%% | `%s` | %s |\n", NR, $1, pct, $2, $3 }' + echo + } >> "$GITHUB_STEP_SUMMARY" + rm -f "$rows" + done + + echo "_Total is the SUM across shards; wall-clock is the slowest shard._" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/tests/run_members.sh b/tests/run_members.sh new file mode 100755 index 00000000..31c1172d --- /dev/null +++ b/tests/run_members.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# run_members.sh — run workspace members one by one, timing each. +# +# The same script CI runs and you run locally, on purpose: a timing table that +# only exists in CI cannot be used while deciding what to optimise, and a local +# harness that differs from CI measures something else. +# +# bash tests/run_members.sh --all +# bash tests/run_members.sh opencv-module protobuf +# bash tests/run_members.sh --all --shard 3/8 +# bash tests/run_members.sh --all --cache local # bypass the package cache +# +# Env: +# MCPP path to the mcpp binary (default: `mcpp` on PATH) +# MCPP_TIMINGS where to append `\t\t` rows +# +# Exit status is non-zero if any member failed. The timing table is printed +# regardless — a slow run is worth measuring even when it breaks. +set -u + +MCPP="${MCPP:-mcpp}" +timings="${MCPP_TIMINGS:-}" +cache="" +shard="" +members=() +all=0 + +while [ $# -gt 0 ]; do + case "$1" in + --all) all=1; shift ;; + --shard) shard="$2"; shift 2 ;; + --cache) cache="$2"; shift 2 ;; + --timings) timings="$2"; shift 2 ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + -*) echo "unknown option: $1" >&2; exit 2 ;; + *) members+=("$1"); shift ;; + esac +done + +# `--all` reads the workspace manifest rather than the directory, so a member +# that exists on disk but is not registered is not silently tested. +# +# Both filters below are load-bearing. mcpp.toml's PROSE mentions the path too +# — line 3 says "tests/examples/ — each consumes this repo's own packages", +# which this grep matches with an empty tail, and `mcpp test -p ""` is not a +# useful thing to run. Requiring a real directory also means a name that only +# appears in a comment (`tests/examples/asio-ssl` is discussed in one) cannot +# turn into a phantom member. +if [ "$all" = 1 ]; then + while IFS= read -r m; do + [ -n "$m" ] || continue + [ -d "tests/examples/$m" ] || continue + members+=("$m") + done < <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml \ + | sed 's|tests/examples/||' | sort -u) +fi + +if [ "${#members[@]}" -eq 0 ]; then + echo "no members selected — pass names or --all" >&2 + exit 2 +fi + +# --shard N/M keeps every M-th member starting at N. Round-robin by position, +# which is what separates adjacent expensive members (opencv-module, +# -dnn, -unifont) onto different runners. +if [ -n "$shard" ]; then + idx=${shard%%/*} + cnt=${shard##*/} + picked=() + i=0 + for m in "${members[@]}"; do + [ $((i % cnt)) -eq "$idx" ] && picked+=("$m") + i=$((i + 1)) + done + members=("${picked[@]+"${picked[@]}"}") + echo "shard $idx/$cnt -> ${#members[@]} member(s)" +fi + +[ -n "$cache" ] && export MCPP_BUILD_CACHE="$cache" +echo "cache mode: ${MCPP_BUILD_CACHE:-global (default)}" + +rows=$(mktemp) +trap 'rm -f "$rows"' EXIT +rc=0 + +for m in "${members[@]}"; do + echo "::group::mcpp test -p $m" + t0=$(date +%s) + if "$MCPP" test -p "$m"; then status=ok; else status=FAIL; rc=1; fi + t1=$(date +%s) + echo "::endgroup::" + printf '%s\t%s\t%s\n' "$((t1 - t0))" "$m" "$status" >> "$rows" + printf ' %-34s %5ss %s\n' "$m" "$((t1 - t0))" "$status" +done + +[ -n "$timings" ] && cat "$rows" >> "$timings" + +echo +echo "── slowest members ──────────────────────────────────────────" +total=$(awk -F'\t' '{s += $1} END {print s+0}' "$rows") +sort -rn "$rows" | head -15 | awk -F'\t' -v tot="$total" ' + { pct = tot > 0 ? ($1 * 100 / tot) : 0 + printf " %6ss %5.1f%% %-34s %s\n", $1, pct, $2, $3 }' +echo " ────────" +printf ' %6ss total across %s member(s)\n' "$total" "${#members[@]}" + +exit "$rc" From e839bace04c1082295bfae5c0fb5260d79c9e734 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 02:24:17 +0800 Subject: [PATCH 06/10] =?UTF-8?q?ci:=20pin=20=E6=8A=AC=E5=88=B0=202026.8.5?= =?UTF-8?q?.4=20=E2=80=94=E2=80=94=20windows=20=E6=94=B9=E7=94=A8=20lld=20?= =?UTF-8?q?=E6=89=8D=E7=9C=9F=E6=AD=A3=E4=BF=AE=E6=8E=89=20LNK1170?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026.8.5.3 把 mcpp **自己**写的响应文件改成按行分隔,必要但不充分:clang 作为 driver 时会**再生成一个**响应文件转发给链接器,那个是单行的,我们改不到。所以 opencv-module 在 .5.3 上仍然 LNK1170(而且是编译完 795s / 1166s 之后)。 2026.8.5.4 让 windows 的 clang 链接改用 lld —— 它的响应文件解析没有单行上限, 消掉的是一整类而不是把数字调大。同时新增了命令长度预算表与计划期校验,超限会在 还没编译任何东西时报出边名与上限,而不是在构建末尾由别人的程序抛一个没有上下文 的错。 本轮全量将同时验证三件事:lld 修复、分片提速、耗时排名。 --- .github/workflows/validate.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 4a0c1745..c172136f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -20,16 +20,19 @@ on: default: global env: - # 2026.8.5.3 carries two things this workflow depends on: + # 2026.8.5.4 carries two things this workflow depends on: # .5.1 `tools = [...]` — how a consumer asks for a dependency's # `kind = "bin"` target, which tests/examples/protobuf-protoc is built # on. Before it: "tools must be a string, inline dep table, or nested # table". - # .5.3 link response files separate objects by NEWLINES. link.exe caps a - # response-file LINE at 128 KiB, and opencv-module / opencv-module-dnn - # went past it on windows — after 795s / 1166s of compiling: + # .5.4 windows links with lld. link.exe caps a response-file LINE at + # 128 KiB and opencv-module / opencv-module-dnn went past it — # fatal error LNK1170: line in command file contains 135135 or # more characters + # after 795s / 1166s of compiling. .5.3 newline-separated OUR response + # file, which was necessary but not sufficient: clang, acting as the + # driver, writes a SECOND one for the linker that we do not control. + # lld's response-file parser has no per-line limit at all. # Together with re-enabling the global package cache below, this is # what makes a green FULL run possible again: .5.3 removes the # windows link failure, the cache removes the 150-minute timeout. @@ -113,7 +116,7 @@ env: # 0.0.94 fixed feature-gated `sources` under `mcpp test` (mcpp#218); 0.0.91 # added standard = "c++fly" to the resolver grammar, so c++fly descriptors # get the lint WARN below, not a hard grammar-parse rejection. - MCPP_VERSION: "2026.8.5.3" + MCPP_VERSION: "2026.8.5.4" jobs: lint: @@ -389,21 +392,21 @@ jobs: ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.5.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION - platform: macos os: macos-15 suffix: macosx-arm64 ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.5.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION - platform: windows os: windows-latest suffix: windows-x86_64 ext: zip mcpp: bin/mcpp.exe xlings: registry/bin/xlings.exe - mcpp_version: "2026.8.5.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION env: MCPP_EFFECTIVE: ${{ matrix.mcpp_version }} steps: From a1b12bb4d94a7f5a4b79152193605028603c04cf Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 02:41:48 +0800 Subject: [PATCH 07/10] =?UTF-8?q?ci:=20=E5=88=86=E7=89=87=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E6=8C=89=E5=AE=9E=E6=B5=8B=E8=80=97=E6=97=B6=E8=A3=85?= =?UTF-8?q?=E7=AE=B1,=E7=89=87=E6=95=B0=E6=8C=89=E5=B9=B3=E5=8F=B0?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E5=BA=A6=20=E2=80=94=E2=80=94=20=E9=98=B2?= =?UTF-8?q?=E6=AD=A2=E3=80=8C=E5=88=86=E4=BA=86=E4=BD=86=E6=B2=A1=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一版分 8 片、位置取模,两处都拍脑袋,实测数据把两处都推翻了。 ── 1. 片数必须匹配平台并发度,不是一个整数 ──────────────────────────────── 实测(24 个 job 排队时): macos 1 · linux 3 · windows 2 合计约 6 并行 墙钟 = ceil(片数 / 并发) × 最慢片。**超出并发的片只增加固定开销**:每片各自 checkout + 下载 mcpp + 恢复缓存(实测 18s)。macOS 并发是 1,分 8 片等于**背靠背 串行跑 8 次**,严格慢于不分片 —— 这就是「分了但没加速」。 改成每平台各自的片数(linux 3 / macos 1 / windows 2),全量才扇出,选择性运行 每平台 1 片。注释里写明这是**实测配额**以及怎么重测。 ── 2. 分配必须看耗时,不能看位置 ────────────────────────────────────────── 取模不知道谁跑得久。实测:它把 ffmpeg、llamacpp-metal、opencv-module-unifont 三个重量级放进同一片,另三片一个都没有 —— 而墙钟只看最慢那片。 新增 tests/plan_shards.lua: - **LPT 装箱**(最长优先 + 放进当前最闲的片),用 tests/member-timings.tsv 的 实测秒数;LPT 对这个问题的近似比是 4/3,再优化不值。 - **依赖亲和**作为近似平局的 tie-break:共享大依赖的成员放同一片就只建一次, 分开就建两次(片间不共享 build cache,只有片内共享)。负载差超过 15% 时 仍以均衡优先 —— 均衡才是墙钟。 - 缺耗时的成员取中位数(新成员既不当免费也不当巨物);**完全没有表时回退取模**, 更差但绝不出错。 用构造数据验证:三个 opencv 被分到三个不同片,负载 16422/15900/15500(差 6%)。 ── 3. 耗时表从哪来 ──────────────────────────────────────────────────────── timings job 现在除了在 run summary 出排名,还产出可直接落库的 `member-timings.tsv` artifact。**刻意不自动提交**:一个每次运行都改写自己的 数字会让每个 diff 都变噪音,还会把一次偶发的慢 runner 悄悄吸收进去。要更新就 下载 artifact 手动替换 —— 数据是自动测的,采纳是人决定的。 本次不附带表(还没有真实数据),所以先走回退路径;等这轮跑完拿到再落库。 --- .github/workflows/validate.yml | 149 ++++++++++++++++++------------- tests/plan_shards.lua | 157 +++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+), 62 deletions(-) create mode 100644 tests/plan_shards.lua diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index c172136f..380f3243 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -272,9 +272,8 @@ jobs: select: runs-on: ubuntu-latest outputs: - members: ${{ steps.fanout.outputs.members }} - shard_count: ${{ steps.fanout.outputs.shard_count }} - shards: ${{ steps.fanout.outputs.shards }} + members: ${{ steps.fanout.outputs.members }} + matrix: ${{ steps.fanout.outputs.matrix }} steps: - uses: actions/checkout@v4 with: @@ -348,67 +347,62 @@ jobs: sel=${sel# } echo "MEMBERS=$sel" >> "$GITHUB_ENV" echo "selected members: ${sel:-}" - # Sharding is for the FULL run only. A selective run is a handful of - # members; splitting it 8 ways would add 7 runners' worth of checkout, - # mcpp download and cache restore to save nothing. + # Sharding is for the FULL run only, and the shard count per platform is + # that platform's RUNNER CONCURRENCY — not a round number. + # + # Measured on this repo (24 jobs queued, 6 running): + # macos 1 · linux 3 · windows 2 + # + # That measurement is what makes over-sharding a real cost rather than a + # theoretical one: at concurrency 1, eight macOS shards run BACK TO BACK + # and each pays its own checkout + mcpp download + cache restore, so the + # split is strictly slower than not splitting. Wall-clock is + # ceil(shards / concurrency) x slowest-shard; shards beyond the + # concurrency only add fixed cost. + # + # Re-measure with: + # gh api repos///actions/runs//jobs --paginate \ + # --jq '[.jobs[]|select(.status=="in_progress")]|length' - name: Decide the fan-out id: fanout shell: bash run: | - if [ "$MEMBERS" = "__ALL__" ]; then - echo 'shards=[0,1,2,3,4,5,6,7]' >> "$GITHUB_OUTPUT" - echo 'shard_count=8' >> "$GITHUB_OUTPUT" - else - echo 'shards=[0]' >> "$GITHUB_OUTPUT" - echo 'shard_count=1' >> "$GITHUB_OUTPUT" - fi + full=0; [ "$MEMBERS" = "__ALL__" ] && full=1 + emit() { # platform os suffix ext mcpp xlings shards + for i in $(seq 0 $(( $7 - 1 ))); do + printf '{"platform":"%s","os":"%s","suffix":"%s","ext":"%s","mcpp":"%s","xlings":"%s","shard":%d,"shards":%d},' \ + "$1" "$2" "$3" "$4" "$5" "$6" "$i" "$7" + done + } + if [ "$full" = 1 ]; then ln=3; mn=1; wn=2; else ln=1; mn=1; wn=1; fi + { + printf '{"include":[' + emit linux ubuntu-latest linux-x86_64 tar.gz bin/mcpp registry/bin/xlings "$ln" + emit macos macos-15 macosx-arm64 tar.gz bin/mcpp registry/bin/xlings "$mn" + emit windows windows-latest windows-x86_64 zip bin/mcpp.exe registry/bin/xlings.exe "$wn" + printf ']}' + } | sed 's/,]}/]}/' > /tmp/matrix.json + echo "matrix=$(cat /tmp/matrix.json)" >> "$GITHUB_OUTPUT" echo "members=$MEMBERS" >> "$GITHUB_OUTPUT" + cat /tmp/matrix.json workspace: - # The shard is in the name only when there is more than one, so a - # selective run still reads "workspace (linux)". - name: workspace (${{ matrix.platform }}${{ needs.select.outputs.shard_count == '1' && '' || format(' {0}/{1}', matrix.shard, needs.select.outputs.shard_count) }}) + # The shard suffix appears only when the platform is actually split. + name: workspace (${{ matrix.platform }}${{ matrix.shards == 1 && '' || format(' {0}/{1}', matrix.shard, matrix.shards) }}) needs: select if: needs.select.outputs.members != '' runs-on: ${{ matrix.os }} - # Was 150 and a full linux run hit it exactly. One shard is ~1/8 of the - # work, so this is now a real ceiling rather than the thing that decides - # whether the job finishes. + # One shard is a fraction of the work, so this is a real ceiling rather + # than the thing that decides whether the job finishes (a full linux run + # used to hit 150 exactly and get cancelled). timeout-minutes: 90 strategy: fail-fast: false - matrix: - # Fanned out ONLY for a full run — `shards` is [0] otherwise, which - # collapses this back to one job per platform. - shard: ${{ fromJSON(needs.select.outputs.shards) }} - platform: [linux, macos, windows] - include: - # Archive names are derived from env.MCPP_VERSION in the Download - # step — bumping the pin is a ONE-line change (hardcoded versions - # here once 404'd a pin bump). - - platform: linux - os: ubuntu-latest - suffix: linux-x86_64 - ext: tar.gz - mcpp: bin/mcpp - xlings: registry/bin/xlings - mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION - - platform: macos - os: macos-15 - suffix: macosx-arm64 - ext: tar.gz - mcpp: bin/mcpp - xlings: registry/bin/xlings - mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION - - platform: windows - os: windows-latest - suffix: windows-x86_64 - ext: zip - mcpp: bin/mcpp.exe - xlings: registry/bin/xlings.exe - mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION + # Whole matrix from `select`: the shard count is per-platform, because it + # tracks that platform's runner concurrency. + matrix: ${{ fromJSON(needs.select.outputs.matrix) }} env: - MCPP_EFFECTIVE: ${{ matrix.mcpp_version }} + MCPP_EFFECTIVE: ${{ env.MCPP_VERSION }} steps: # Full history: the member-selection step below diffs against the PR # base to decide which workspace members to test. @@ -485,23 +479,28 @@ jobs: # opencv-module / -dnn / -unifont are adjacent in the list, so `% N` # necessarily puts them on three different runners. A single job that # builds all three spends 45+ minutes on opencv alone. + # ── This shard's slice ──────────────────────────────────────────── + # `select` decided WHAT runs; tests/plan_shards.lua decides which part + # runs HERE — by MEASURED time (tests/member-timings.tsv, produced by the + # timings job below), longest-first, with dependency affinity breaking + # near-ties so members sharing a big dependency land together and build + # it once. Round-robin by position was the first version and balance was + # luck: it put ffmpeg, llamacpp-metal and opencv-module-unifont on one + # shard and nothing heavy on three others, and wall-clock is the slowest + # shard. - name: Take this shard's members shell: bash run: | - plan='${{ needs.select.outputs.members }}' - if [ "$plan" = "__ALL__" ]; then - plan=$(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml \ - | sed 's|tests/examples/||' | sort -u | tr '\n' ' ') + if ! command -v lua5.4 >/dev/null 2>&1; then + sudo apt-get install -y --no-install-recommends lua5.4 >/dev/null 2>&1 \ + || brew install lua >/dev/null 2>&1 || true fi - n=${{ needs.select.outputs.shard_count }} - mine=""; i=0 - for m in $plan; do - [ $((i % n)) -eq ${{ matrix.shard }} ] && mine="$mine $m" - i=$((i + 1)) - done - mine=${mine# } + plan='${{ needs.select.outputs.members }}' + if [ "$plan" = "__ALL__" ]; then plan=""; fi + mine=$(lua5.4 tests/plan_shards.lua "${{ matrix.platform }}" \ + "${{ matrix.shard }}" "${{ matrix.shards }}" $plan) echo "MEMBERS=$mine" >> "$GITHUB_ENV" - echo "shard ${{ matrix.shard }}/$n of $i member(s): ${mine:-}" + echo "shard ${{ matrix.shard }}/${{ matrix.shards }}: ${mine:-}" # ── Refresh the PUBLISHED index before testing ──────────────────── # Most members resolve everything from this checkout, but a member that @@ -655,3 +654,29 @@ jobs: echo "_Total is the SUM across shards; wall-clock is the slowest shard._" \ >> "$GITHUB_STEP_SUMMARY" + + # The table that feeds the NEXT run's sharding. Emitted as an + # artifact rather than committed automatically: a number that + # rewrites itself on every run would make every diff noisy and would + # silently absorb a one-off slow runner. Refresh it deliberately — + # download this artifact and replace tests/member-timings.tsv when + # the numbers have actually moved. + { + echo "# \t\t — from run ${{ github.run_id }}" + echo "# refresh: download the member-timings artifact and replace this file" + for plat in linux macos windows; do + for f in timings/timings-$plat-*/timings.tsv; do + [ -f "$f" ] || continue + awk -F'\t' -v p="$plat" '{ printf "%s\t%s\t%s\n", p, $2, $1 }' "$f" + done + done + } | sort -u > member-timings.tsv + echo "wrote member-timings.tsv ($(grep -vc '^#' member-timings.tsv) rows)" + + - name: Upload the timing table for the next run's sharding + if: always() && hashFiles('member-timings.tsv') != '' + uses: actions/upload-artifact@v4 + with: + name: member-timings + path: member-timings.tsv + retention-days: 90 diff --git a/tests/plan_shards.lua b/tests/plan_shards.lua new file mode 100644 index 00000000..e6cb7874 --- /dev/null +++ b/tests/plan_shards.lua @@ -0,0 +1,157 @@ +#!/usr/bin/env lua5.4 +-- plan_shards.lua — decide which members run on which shard. +-- +-- lua5.4 tests/plan_shards.lua [member...] +-- +-- Prints this shard's members, space-separated. With no member arguments it +-- reads the whole workspace from mcpp.toml. +-- +-- WHY NOT ROUND-ROBIN BY POSITION +-- +-- The first version split `i % count`, which knows nothing about how long +-- anything takes; balance was luck. Measured on the real workspace it put +-- three heavyweights (ffmpeg, llamacpp-metal, opencv-module-unifont) on one +-- shard and none on three others — and wall-clock is the SLOWEST shard, so the +-- idle ones bought nothing. +-- +-- Two things decide the split here instead: +-- +-- 1. MEASURED TIME (tests/member-timings.tsv, produced by CI). Longest- +-- Processing-Time first: sort descending, put each member on the shard +-- with the least load so far. LPT is within 4/3 of optimal for this +-- problem, and optimal is not worth more than that here. +-- 2. DEPENDENCY AFFINITY, as the tie-break. Two members that share a +-- dependency build it once if they land on the same shard and twice if +-- they do not — shards do not share a build cache, only a run does. So +-- among shards whose load is close, prefer the one already holding +-- members with overlapping dependencies. +-- +-- Missing timing → the median, so a newly added member is neither assumed +-- free nor assumed huge. No table at all → falls back to round-robin, which +-- is worse but never wrong. + +local platform = arg[1] or error("usage: plan_shards.lua [members...]") +local shardIndex = tonumber(arg[2]) or error("shard index must be a number") +local shardCount = tonumber(arg[3]) or error("shard count must be a number") + +local members = {} +for i = 4, #arg do members[#members + 1] = arg[i] end + +local function read_file(path) + local f = io.open(path, "r"); if not f then return nil end + local s = f:read("a"); f:close(); return s +end + +-- Whole workspace, from the manifest rather than the directory: a member that +-- exists on disk but is not registered must not be tested. Both filters below +-- matter — mcpp.toml's prose mentions `tests/examples/` too, which yields an +-- empty name, and a name that only appears in a comment is not a member. +if #members == 0 then + local toml = read_file("mcpp.toml") or error("cannot read mcpp.toml") + local seen = {} + for name in toml:gmatch("tests/examples/([A-Za-z0-9._%-]+)") do + if name ~= "" and not seen[name] then + local probe = io.open("tests/examples/" .. name .. "/mcpp.toml", "r") + if probe then probe:close(); seen[name] = true; members[#members + 1] = name end + end + end + table.sort(members) +end + +if shardCount <= 1 then + print(table.concat(members, " ")) + return +end + +-- ── measured times ──────────────────────────────────────────────────────── +-- Format: \t\t +local times, samples = {}, {} +local tsv = read_file("tests/member-timings.tsv") +if tsv then + for line in tsv:gmatch("[^\n]+") do + if not line:match("^#") then + local p, m, s = line:match("^(%S+)\t(%S+)\t(%d+)") + if p == platform and m then + times[m] = tonumber(s) + samples[#samples + 1] = tonumber(s) + end + end + end +end + +local median = 60 +if #samples > 0 then + table.sort(samples) + median = samples[math.ceil(#samples / 2)] +end + +-- ── dependency signature, for affinity ──────────────────────────────────── +local function deps_of(member) + local toml = read_file("tests/examples/" .. member .. "/mcpp.toml") + if not toml then return {} end + local set = {} + -- The package names a member depends on. Deliberately crude: exact + -- accuracy is not needed, only "do these two pull the same big things". + for name in toml:gmatch("\n%s*([A-Za-z][A-Za-z0-9._%-]*)%s*=") do + if name ~= "name" and name ~= "version" and name ~= "standard" + and name ~= "sources" and name ~= "kind" and name ~= "main" + and name ~= "description" and name ~= "license" then + set[name] = true + end + end + return set +end + +local depsCache = {} +local function deps(member) + if depsCache[member] == nil then depsCache[member] = deps_of(member) end + return depsCache[member] +end + +-- ── LPT with affinity tie-break ─────────────────────────────────────────── +local ordered = {} +for _, m in ipairs(members) do ordered[#ordered + 1] = m end +table.sort(ordered, function(a, b) + local ta, tb = times[a] or median, times[b] or median + if ta ~= tb then return ta > tb end + return a < b -- deterministic across machines +end) + +local shards = {} +for i = 0, shardCount - 1 do shards[i] = { load = 0, members = {}, deps = {} } end + +for _, m in ipairs(ordered) do + local cost = times[m] or median + local best, bestLoad, bestAffinity = nil, nil, -1 + for i = 0, shardCount - 1 do + local s = shards[i] + -- Affinity only breaks near-ties: a shard 15% lighter always wins, + -- because balance is what wall-clock actually measures. + local affinity = 0 + for d in pairs(deps(m)) do if s.deps[d] then affinity = affinity + 1 end end + if best == nil then + best, bestLoad, bestAffinity = i, s.load, affinity + else + local margin = math.max(bestLoad, s.load) * 0.15 + if s.load < bestLoad - margin then + best, bestLoad, bestAffinity = i, s.load, affinity + elseif math.abs(s.load - bestLoad) <= margin and affinity > bestAffinity then + best, bestLoad, bestAffinity = i, s.load, affinity + end + end + end + local s = shards[best] + s.load = s.load + cost + s.members[#s.members + 1] = m + for d in pairs(deps(m)) do s.deps[d] = true end +end + +if os.getenv("PLAN_SHARDS_DEBUG") then + for i = 0, shardCount - 1 do + io.stderr:write(string.format("shard %d: load=%ds n=%d %s\n", + i, shards[i].load, #shards[i].members, + table.concat(shards[i].members, " "))) + end +end + +print(table.concat(shards[shardIndex] and shards[shardIndex].members or {}, " ")) From 6eb2c9f4bfe56fab583f562079b11eb3ea4d5309 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 03:36:58 +0800 Subject: [PATCH 08/10] =?UTF-8?q?ci:=20=E8=90=BD=E5=BA=93=E5=AE=9E?= =?UTF-8?q?=E6=B5=8B=E8=80=97=E6=97=B6=E8=A1=A8=20=E2=80=94=E2=80=94=20LPT?= =?UTF-8?q?=20=E6=AF=94=E5=8F=96=E6=A8=A1=E7=9C=81=207.5=20=E5=88=86?= =?UTF-8?q?=E9=92=9F=E5=A2=99=E9=92=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 用真实数据(run 31034885938,24 个分片的 timings artifact)替换掉「无表回退」。 183 行实测,linux 分 3 片的对照: 取模 4158 / 3027 / 2822 最慢 4158s LPT 3706 / 3152 / 3149 最慢 3706s 墙钟只看最慢片,所以 **省 452s(7.5 分钟,11%)**,极差从 47% 降到 15%。 顺带一个决定继续不继续加片数的数字:**单个最慢成员是硬下界**。grpc-module 一个 就 1701s(28 分钟),linux 再怎么分也快不过它。要再快只能让那个成员本身变快, 不是加 runner。 表由 CI 自动产出,采纳由人做(见 workflow 里的说明):自动提交会让每个 diff 变 噪音,还会把一次偶发的慢 runner 悄悄吸收进去。 --- tests/member-timings.tsv | 185 +++++++++++++++++++++++++++++++++++++++ tests/plan_shards.lua | 11 +++ 2 files changed, 196 insertions(+) create mode 100644 tests/member-timings.tsv diff --git a/tests/member-timings.tsv b/tests/member-timings.tsv new file mode 100644 index 00000000..2c8a912e --- /dev/null +++ b/tests/member-timings.tsv @@ -0,0 +1,185 @@ +linux abseil 111 +linux archive 49 +linux asio-module 37 +linux asio-ssl 49 +linux boost-ext.ut 30 +linux build-mcpp 28 +linux c-ares 28 +linux catch2 81 +linux catch2-main 53 +linux catch2-v2 13 +linux catch2-v2-main 15 +linux cjson 2 +linux core 67 +linux curl 26 +linux eigen 25 +linux eui-neo 151 +linux eui-neo-app-main 140 +linux eui-neo-markdown 143 +linux eui-neo-sdl2 119 +linux eui-neo-vulkan 154 +linux eui-neo-window 128 +linux ffmpeg 288 +linux ffmpeg-module 343 +linux fmtlib.fmt 5 +linux freetype 10 +linux glad 2 +linux godot-cpp 573 +linux godot-cpp-module 549 +linux godot-cpp-module-v10 392 +linux godot-cpp-v10 504 +linux grpc-module 1701 +linux gui-stack 81 +linux imgui 6 +linux imgui-module 91 +linux imgui-window 91 +linux libpng 6 +linux llamacpp 101 +linux llamacpp-metal 0 +linux magic_enum 4 +linux marzer.tomlplusplus 7 +linux md4c 3 +linux nlohmann.json 10 +linux openblas 0 +linux opencv-module 602 +linux opencv-module-dnn 741 +linux opencv-module-unifont 671 +linux openssl 0 +linux protobuf 259 +linux protobuf-gzip 158 +linux protobuf-protoc 923 +linux protobuf-upb 263 +linux re2 15 +linux sdl2 63 +linux spdlog 9 +linux spdlog-compiled 13 +linux tinyhttps 13 +linux tray 3 +linux vulkan 13 +linux websocket 19 +linux websocket-features 23 +linux yyjson 3 +macos abseil 75 +macos archive 49 +macos asio-module 30 +macos asio-ssl 78 +macos boost-ext.ut 27 +macos build-mcpp 33 +macos c-ares 27 +macos catch2 58 +macos catch2-main 28 +macos catch2-v2 10 +macos catch2-v2-main 9 +macos cjson 3 +macos core 58 +macos curl 103 +macos eigen 7 +macos eui-neo 52 +macos eui-neo-app-main 57 +macos eui-neo-markdown 76 +macos eui-neo-sdl2 142 +macos eui-neo-vulkan 59 +macos eui-neo-window 53 +macos ffmpeg 148 +macos ffmpeg-module 130 +macos fmtlib.fmt 4 +macos freetype 12 +macos glad 3 +macos godot-cpp 333 +macos godot-cpp-module 256 +macos godot-cpp-module-v10 215 +macos godot-cpp-v10 230 +macos grpc-module 880 +macos gui-stack 1 +macos imgui 4 +macos imgui-module 1 +macos imgui-window 1 +macos libpng 7 +macos llamacpp 66 +macos llamacpp-metal 75 +macos magic_enum 3 +macos marzer.tomlplusplus 5 +macos md4c 4 +macos nlohmann.json 9 +macos openblas 1 +macos opencv-module 198 +macos opencv-module-dnn 399 +macos opencv-module-unifont 304 +macos openssl 1 +macos protobuf 100 +macos protobuf-gzip 87 +macos protobuf-protoc 458 +macos protobuf-upb 143 +macos re2 8 +macos sdl2 22 +macos spdlog 5 +macos spdlog-compiled 6 +macos tinyhttps 13 +macos tray 4 +macos vulkan 7 +macos websocket 12 +macos websocket-features 12 +macos yyjson 6 +# — measured, run 31034885938 +# refresh: download the member-timings artifact from a full run and replace this file +windows abseil 133 +windows archive 76 +windows asio-module 37 +windows asio-ssl 24 +windows boost-ext.ut 40 +windows build-mcpp 27 +windows c-ares 33 +windows catch2 83 +windows catch2-main 62 +windows catch2-v2 12 +windows catch2-v2-main 12 +windows cjson 3 +windows core 98 +windows curl 31 +windows eigen 10 +windows eui-neo 93 +windows eui-neo-app-main 119 +windows eui-neo-markdown 84 +windows eui-neo-sdl2 150 +windows eui-neo-vulkan 84 +windows eui-neo-window 74 +windows ffmpeg 454 +windows ffmpeg-module 532 +windows fmtlib.fmt 5 +windows freetype 15 +windows glad 2 +windows godot-cpp 742 +windows godot-cpp-module 636 +windows godot-cpp-module-v10 684 +windows godot-cpp-v10 566 +windows grpc-module 1 +windows gui-stack 1 +windows imgui 6 +windows imgui-module 0 +windows imgui-window 0 +windows libpng 10 +windows llamacpp 125 +windows llamacpp-metal 1 +windows magic_enum 4 +windows marzer.tomlplusplus 6 +windows md4c 3 +windows nlohmann.json 10 +windows openblas 11 +windows opencv-module 747 +windows opencv-module-dnn 1174 +windows opencv-module-unifont 1 +windows openssl 1 +windows protobuf 215 +windows protobuf-gzip 141 +windows protobuf-protoc 227 +windows protobuf-upb 251 +windows re2 15 +windows sdl2 46 +windows spdlog 6 +windows spdlog-compiled 14 +windows tinyhttps 14 +windows tray 2 +windows vulkan 7 +windows websocket 24 +windows websocket-features 26 +windows yyjson 2 diff --git a/tests/plan_shards.lua b/tests/plan_shards.lua index e6cb7874..e92f8665 100644 --- a/tests/plan_shards.lua +++ b/tests/plan_shards.lua @@ -29,6 +29,17 @@ -- Missing timing → the median, so a newly added member is neither assumed -- free nor assumed huge. No table at all → falls back to round-robin, which -- is worse but never wrong. +-- +-- Measured on the real workspace (linux, 3 shards), LPT against round-robin: +-- +-- round-robin 4158 / 3027 / 2822 slowest 4158s +-- LPT 3706 / 3152 / 3149 slowest 3706s +-- +-- 452s off the wall-clock, and the spread drops from 47% to 15%. +-- +-- There is a floor no split can beat: the single slowest member. grpc-module +-- alone is 1701s, so linux cannot finish faster than that however many shards +-- there are — which is the number to look at before adding more. local platform = arg[1] or error("usage: plan_shards.lua [members...]") local shardIndex = tonumber(arg[2]) or error("shard index must be a number") From 0503f1acc3bc217a28b9c69b9db9b0998c6e9cee Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 03:43:49 +0800 Subject: [PATCH 09/10] =?UTF-8?q?fix(ci):=20job=20=E7=BA=A7=20env=20?= =?UTF-8?q?=E4=B8=8D=E8=83=BD=E5=BC=95=E7=94=A8=20env=20=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87,=E5=AF=BC=E8=87=B4=20workflow=20=E4=B8=80=E4=B8=AA=20?= =?UTF-8?q?job=20=E9=83=BD=E4=B8=8D=E5=90=AF=E5=8A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一个 commit 里写了: env: MCPP_EFFECTIVE: ${{ env.MCPP_VERSION }} **job 级 `env:` 里不能引用 `env` 上下文**。workflow 在校验阶段就失败,run 创建出来但 `jobs` 是空的、日志 "log not found" —— 和「某个 job 挂了」完全是两回事, 排查方向也完全不同。 MCPP_EFFECTIVE 原本只是 matrix.mcpp_version 的别名;矩阵改由 select 生成之后 这层间接没有存在的理由了,直接全部用 MCPP_VERSION。 顺带记一条判别法:**run 有了但 jobs 为空 = workflow 启动期失败**(YAML schema 或表达式求值),不是任何一个 job 的问题;而 "no checks reported" 则通常是 PR 有冲突、GitHub 算不出 merge ref。两种都表现为「CI 好像没跑」。 --- .github/workflows/validate.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 380f3243..11f2ec06 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -401,8 +401,6 @@ jobs: # Whole matrix from `select`: the shard count is per-platform, because it # tracks that platform's runner concurrency. matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - env: - MCPP_EFFECTIVE: ${{ env.MCPP_VERSION }} steps: # Full history: the member-selection step below diffs against the PR # base to decide which workspace members to test. @@ -432,7 +430,7 @@ jobs: # only a Git-Bash convenience. h=$(git ls-files -s -- 'pkgs/**/*.lua' 'tests/**' '.github/workflows/validate.yml' \ | git hash-object --stdin) - echo "REGISTRY_CACHE_KEY=mcpp-registry-${{ runner.os }}-${{ env.MCPP_EFFECTIVE }}-$h" >> "$GITHUB_ENV" + echo "REGISTRY_CACHE_KEY=mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}-$h" >> "$GITHUB_ENV" - name: Restore mcpp registry cache uses: actions/cache@v4 with: @@ -441,15 +439,15 @@ jobs: path: ~/.mcpp/registry key: ${{ env.REGISTRY_CACHE_KEY }} restore-keys: | - mcpp-registry-${{ runner.os }}-${{ env.MCPP_EFFECTIVE }}- + mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}- - name: Download mcpp shell: bash env: - MCPP_ARCHIVE: mcpp-${{ env.MCPP_EFFECTIVE }}-${{ matrix.suffix }}.${{ matrix.ext }} - MCPP_ROOT: mcpp-${{ env.MCPP_EFFECTIVE }}-${{ matrix.suffix }} + MCPP_ARCHIVE: mcpp-${{ env.MCPP_VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }} + MCPP_ROOT: mcpp-${{ env.MCPP_VERSION }}-${{ matrix.suffix }} run: | curl -L -fsS -o "$MCPP_ARCHIVE" \ - "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_EFFECTIVE}/${MCPP_ARCHIVE}" + "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_VERSION}/${MCPP_ARCHIVE}" case "$MCPP_ARCHIVE" in *.zip) powershell -NoProfile -Command "Expand-Archive -Force -Path '${MCPP_ARCHIVE}' -DestinationPath '.'" ;; *) tar -xzf "$MCPP_ARCHIVE" ;; From 3ce4bc533fe6aacde2c8712d143a6e6af0f021c6 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 6 Aug 2026 03:47:51 +0800 Subject: [PATCH 10/10] =?UTF-8?q?fix(ci):=20=E5=88=86=E7=89=87=E8=AE=A1?= =?UTF-8?q?=E5=88=92=E6=8C=AA=E5=88=B0=20select=20=E7=AE=97=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=20=E2=80=94=E2=80=94=20runner=20=E4=B8=8A=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=20lua5.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macos / windows 的分片在 16 秒就挂了: lua5.4: command not found (exit 127) 我把 plan_shards.lua 放在每个 runner 上跑,而 lua5.4 只在 ubuntu 的 lint job 里装过。写的回退也不成立:**windows 没有 apt 也没有 brew**,而 **macOS 的 brew 装的是 `lua`,不是 `lua5.4`**。 改成在 select job(ubuntu,本来就装 lua)里**算一次**,把每片的成员列表作为 数据发给 runner。runner 侧不再需要 lua。 这不只是修 bug —— 决策本来就该集中在一处。让三个 runner 各自重新推导同一个 答案,正是本仓库反复付学费的那个形状。 本地用同样的 jq/lua 管线模拟过:JSON 合法,linux 22/18/21、windows 33/28、 macos 61(并发为 1 故不分片)。 --- .github/workflows/validate.yml | 56 ++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 11f2ec06..f12cc10e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -274,10 +274,13 @@ jobs: outputs: members: ${{ steps.fanout.outputs.members }} matrix: ${{ steps.fanout.outputs.matrix }} + plan: ${{ steps.plan_shards.outputs.plan }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Install lua + run: sudo apt-get install -y --no-install-recommends lua5.4 # ── Selective member testing ────────────────────────────────────── # `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and # dominates CI wall-clock, while a PR almost always touches one @@ -386,6 +389,40 @@ jobs: echo "members=$MEMBERS" >> "$GITHUB_OUTPUT" cat /tmp/matrix.json + # The split is computed ONCE, here, and shipped to the runners as data. + # It used to run on each runner, which needed lua5.4 on all three + # platforms — windows has no apt or brew, and macOS's brew installs + # `lua`, not `lua5.4`, so every non-linux shard died with + # `lua5.4: command not found` after 16 seconds. Deciding once is also + # simply correct: one plan, not three runners each re-deriving it. + - name: Plan the shards + id: plan_shards + shell: bash + run: | + plan='${{ steps.fanout.outputs.members }}' + [ "$plan" = "__ALL__" ] && plan="" + { + printf '{' + first=1 + for spec in linux:$(jq -r '[.include[]|select(.platform=="linux")]|length' /tmp/matrix.json) \ + macos:$(jq -r '[.include[]|select(.platform=="macos")]|length' /tmp/matrix.json) \ + windows:$(jq -r '[.include[]|select(.platform=="windows")]|length' /tmp/matrix.json); do + p=${spec%%:*}; n=${spec##*:} + [ "$first" = 1 ] || printf ',' + first=0 + printf '"%s":{' "$p" + for i in $(seq 0 $((n - 1))); do + [ "$i" = 0 ] || printf ',' + m=$(lua5.4 tests/plan_shards.lua "$p" "$i" "$n" $plan) + printf '"%s":"%s"' "$i" "$m" + done + printf '}' + done + printf '}' + } > /tmp/plan.json + echo "plan=$(cat /tmp/plan.json)" >> "$GITHUB_OUTPUT" + jq . /tmp/plan.json + workspace: # The shard suffix appears only when the platform is actually split. name: workspace (${{ matrix.platform }}${{ matrix.shards == 1 && '' || format(' {0}/{1}', matrix.shard, matrix.shards) }}) @@ -478,25 +515,12 @@ jobs: # necessarily puts them on three different runners. A single job that # builds all three spends 45+ minutes on opencv alone. # ── This shard's slice ──────────────────────────────────────────── - # `select` decided WHAT runs; tests/plan_shards.lua decides which part - # runs HERE — by MEASURED time (tests/member-timings.tsv, produced by the - # timings job below), longest-first, with dependency affinity breaking - # near-ties so members sharing a big dependency land together and build - # it once. Round-robin by position was the first version and balance was - # luck: it put ffmpeg, llamacpp-metal and opencv-module-unifont on one - # shard and nothing heavy on three others, and wall-clock is the slowest - # shard. + # Already decided by `select` (measured-time bin packing, see + # tests/plan_shards.lua). Arrives as data, so a runner needs no lua. - name: Take this shard's members shell: bash run: | - if ! command -v lua5.4 >/dev/null 2>&1; then - sudo apt-get install -y --no-install-recommends lua5.4 >/dev/null 2>&1 \ - || brew install lua >/dev/null 2>&1 || true - fi - plan='${{ needs.select.outputs.members }}' - if [ "$plan" = "__ALL__" ]; then plan=""; fi - mine=$(lua5.4 tests/plan_shards.lua "${{ matrix.platform }}" \ - "${{ matrix.shard }}" "${{ matrix.shards }}" $plan) + mine='${{ fromJSON(needs.select.outputs.plan)[matrix.platform][format('{0}', matrix.shard)] }}' echo "MEMBERS=$mine" >> "$GITHUB_ENV" echo "shard ${{ matrix.shard }}/${{ matrix.shards }}: ${mine:-}"