From 341d8617fa324c002cdbbfe7e32adcd778cbce2c Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 09:48:53 +0000 Subject: [PATCH 1/2] fix(TC-887): refresh the w5 lockfile and check separate workspace lockfiles in CI test/w5-policy-runtime-node-e2e/Cargo.lock lacked tinycloud-core's new aes, hkdf and zeroize dependencies and the aes/aes-gcm zeroize features (#320, #322), so cargo --locked failed there. Regenerated minimally. CI built only the main workspace and m1-realdata-e2e. A new lockfiles job resolves every fetchable workspace with --locked and runs scripts/ci/check-separate-lockfiles.py, which checks the w5 lockfile (its private git dependency is not fetchable in CI) for in-repo crates' direct dependencies and requested dependency features. --- .github/workflows/rust.yml | 30 ++++- scripts/ci/check-separate-lockfiles.py | 126 +++++++++++++++++++++ test/w5-policy-runtime-node-e2e/Cargo.lock | 5 + 3 files changed, 159 insertions(+), 2 deletions(-) create mode 100755 scripts/ci/check-separate-lockfiles.py diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d3ca3802..0fc0ba28 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -208,8 +208,9 @@ jobs: # repository by git, which this workflow's token cannot clone, so it runs # locally before a release (`cargo test --locked --manifest-path # test/w5-policy-runtime-node-e2e/Cargo.toml`); a CI job needs a read token - # for that repository first. test/load/signer is covered by no job at all. - # Recorded here rather than left implicit. + # for that repository first. test/load/signer is built by no job. The + # `lockfiles` job below keeps all three lockfiles current. Recorded here + # rather than left implicit. m1-realdata: runs-on: ubuntu-latest env: @@ -239,6 +240,31 @@ jobs: - name: Run deterministic M1 contract tests run: cargo test --locked --manifest-path test/m1-realdata-e2e/Cargo.toml + # TC-887: every separate workspace's Cargo.lock must stay current for the + # in-repo crates it uses by path, or `cargo ... --locked` breaks there. The + # jobs above build only the main workspace and m1-realdata-e2e. This job + # resolves each workspace CI can fetch with --locked, and checks the w5 + # lockfile (whose private git dependency CI cannot fetch) without network: + # in-repo crates' direct dependencies and requested dependency features. + lockfiles: + name: separate workspace lockfiles + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + steps: + - name: Checkout TinyCloud repository + uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.97.1 + - name: Resolve each fetchable workspace with --locked + run: | + for manifest in Cargo.toml test/m1-realdata-e2e/Cargo.toml test/load/signer/Cargo.toml; do + echo "cargo metadata --locked --manifest-path $manifest" + cargo metadata --locked --format-version 1 --manifest-path "$manifest" > /dev/null + done + - name: Check separate lockfiles against in-repo crates + run: python3 scripts/ci/check-separate-lockfiles.py + # TC-887 S0: a real node on native PostgreSQL and native MinIO (no service # containers). MinIO/mc are pinned community releases whose sha256 is pinned # in harness.sh; PostgreSQL is the runner image's own native install. The diff --git a/scripts/ci/check-separate-lockfiles.py b/scripts/ci/check-separate-lockfiles.py new file mode 100755 index 00000000..b10f2a9f --- /dev/null +++ b/scripts/ci/check-separate-lockfiles.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Check that every separate Cargo workspace's lockfile is current for the +workspace crates it uses by path (TC-887). + +Some workspaces (test/w5-policy-runtime-node-e2e, test/m1-realdata-e2e, +test/load/signer) are excluded from the main workspace and carry their own +Cargo.lock. When a workspace crate such as tinycloud-core gains a dependency +or a dependency feature, the main Cargo.lock is updated by the build, but +those lockfiles are not, and `cargo ... --locked` then fails in them. CI +cannot run a `--locked` resolve of w5 (it depends on a private repository), +so this check needs no network. For every in-repo crate a separate lockfile +records (per `cargo metadata --no-deps` of the main workspace): +- each normal, non-optional dependency must appear in its lockfile entry; +- each feature it requests on a dependency that, in the main Cargo.lock, + pulls in a package (e.g. `aes` feature `zeroize`) must pull in that package + in the separate lockfile too. +Workspaces that CI can resolve are also checked with `cargo metadata +--locked` in the `lockfiles` job. + +Usage: scripts/ci/check-separate-lockfiles.py (from the repository root) +""" +import json +import pathlib +import re +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SKIP_DIRS = {"target", "node_modules", ".git"} + + +def lock_entries(path): + """Yield (name, has_source, dependency names) for each [[package]].""" + for block in path.read_text().split("[[package]]")[1:]: + name = re.search(r'^name = "(.*)"$', block, re.M).group(1) + has_source = re.search(r"^source = ", block, re.M) is not None + deps = [] + if "dependencies = [" in block: + body = block.split("dependencies = [", 1)[1].split("]", 1)[0] + deps = re.findall(r'"([^" ]+)', body) + yield name, has_source, deps + + +def path_packages(path): + """In-repo (source-less) package name -> its dependency names.""" + return {n: set(d) for n, has_source, d in lock_entries(path) if not has_source} + + +def all_packages(path): + """Package name -> dependency names, merged over every entry of that name.""" + merged = {} + for name, _, deps in lock_entries(path): + merged.setdefault(name, set()).update(deps) + return merged + + +def workspace(): + """In-repo crates (name -> (normal deps, {dep: features})) and member dirs.""" + meta = json.loads( + subprocess.check_output( + ["cargo", "metadata", "--no-deps", "--offline", "--format-version", "1"], + cwd=ROOT, + ) + ) + crates = {} + for pkg in meta["packages"]: + normal = [d for d in pkg["dependencies"] if d["kind"] is None] + crates[pkg["name"]] = ( + sorted({d["name"] for d in normal if not d["optional"]}), + {d["name"]: d["features"] for d in normal if d["features"]}, + ) + members = { + pathlib.Path(pkg["manifest_path"]).parent + for pkg in meta["packages"] + if pkg["id"] in meta["workspace_members"] + } + return crates, members + + +def separate_lockfiles(members): + for lock in sorted(ROOT.rglob("Cargo.lock")): + rel = lock.relative_to(ROOT) + if rel == pathlib.Path("Cargo.lock") or SKIP_DIRS & set(rel.parts): + continue + if lock.parent in members: + # A main-workspace member's leftover lockfile: cargo never reads it. + continue + yield lock + + +def main(): + crates, members = workspace() + root_all = all_packages(ROOT / "Cargo.lock") + failures, checked = [], 0 + for lock in separate_lockfiles(members): + rel = lock.relative_to(ROOT) + recorded = path_packages(lock) + everything = all_packages(lock) + for name, (needed, features) in crates.items(): + if name not in recorded: + continue + checked += 1 + missing = [d for d in needed if d not in recorded[name]] + if missing: + failures.append(f"{rel}: {name} is missing {', '.join(missing)}") + for dep, wanted in features.items(): + for feature in wanted: + if feature in root_all.get(dep, ()) and feature not in everything.get(dep, ()): + failures.append( + f"{rel}: {dep} lacks {feature} (feature `{feature}` requested by {name})" + ) + for f in failures: + print(f"stale lockfile: {f}", file=sys.stderr) + if failures: + print( + "Refresh each listed lockfile, e.g. " + "`cargo update -p --manifest-path /Cargo.toml`.", + file=sys.stderr, + ) + return 1 + print(f"separate lockfiles current ({checked} in-repo crate entries checked)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/w5-policy-runtime-node-e2e/Cargo.lock b/test/w5-policy-runtime-node-e2e/Cargo.lock index 431ccf7c..29f068d0 100644 --- a/test/w5-policy-runtime-node-e2e/Cargo.lock +++ b/test/w5-policy-runtime-node-e2e/Cargo.lock @@ -46,6 +46,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -60,6 +61,7 @@ dependencies = [ "ctr", "ghash", "subtle", + "zeroize", ] [[package]] @@ -8182,6 +8184,7 @@ dependencies = [ name = "tinycloud-core" version = "1.3.0" dependencies = [ + "aes", "aes-gcm", "arrow", "async-trait", @@ -8191,6 +8194,7 @@ dependencies = [ "duckdb", "futures", "hex", + "hkdf", "hmac", "k256", "libp2p", @@ -8219,6 +8223,7 @@ dependencies = [ "unicode-normalization", "uuid", "x25519-dalek", + "zeroize", ] [[package]] From ae6d4d98c6689cb3d52b772144b8f94cb2551a41 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 09:54:50 +0000 Subject: [PATCH 2/2] ci(TC-887): drop the hand-rolled w5 lockfile checker Keep the w5 lockfile fix and the --locked resolves of main, m1 and test/load/signer. w5 coverage follows separately with real cargo. --- .github/workflows/rust.yml | 12 +-- scripts/ci/check-separate-lockfiles.py | 126 ------------------------- 2 files changed, 5 insertions(+), 133 deletions(-) delete mode 100755 scripts/ci/check-separate-lockfiles.py diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0fc0ba28..90de2f53 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -208,8 +208,9 @@ jobs: # repository by git, which this workflow's token cannot clone, so it runs # locally before a release (`cargo test --locked --manifest-path # test/w5-policy-runtime-node-e2e/Cargo.toml`); a CI job needs a read token - # for that repository first. test/load/signer is built by no job. The - # `lockfiles` job below keeps all three lockfiles current. Recorded here + # for that repository first, and its lockfile is not checked in CI yet. + # test/load/signer is built by no job. The `lockfiles` job below resolves + # main, m1-realdata-e2e and test/load/signer with --locked. Recorded here # rather than left implicit. m1-realdata: runs-on: ubuntu-latest @@ -243,9 +244,8 @@ jobs: # TC-887: every separate workspace's Cargo.lock must stay current for the # in-repo crates it uses by path, or `cargo ... --locked` breaks there. The # jobs above build only the main workspace and m1-realdata-e2e. This job - # resolves each workspace CI can fetch with --locked, and checks the w5 - # lockfile (whose private git dependency CI cannot fetch) without network: - # in-repo crates' direct dependencies and requested dependency features. + # resolves each workspace CI can fetch with --locked (w5's private git + # dependency is not fetchable here; see above). lockfiles: name: separate workspace lockfiles runs-on: ubuntu-latest @@ -262,8 +262,6 @@ jobs: echo "cargo metadata --locked --manifest-path $manifest" cargo metadata --locked --format-version 1 --manifest-path "$manifest" > /dev/null done - - name: Check separate lockfiles against in-repo crates - run: python3 scripts/ci/check-separate-lockfiles.py # TC-887 S0: a real node on native PostgreSQL and native MinIO (no service # containers). MinIO/mc are pinned community releases whose sha256 is pinned diff --git a/scripts/ci/check-separate-lockfiles.py b/scripts/ci/check-separate-lockfiles.py deleted file mode 100755 index b10f2a9f..00000000 --- a/scripts/ci/check-separate-lockfiles.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -"""Check that every separate Cargo workspace's lockfile is current for the -workspace crates it uses by path (TC-887). - -Some workspaces (test/w5-policy-runtime-node-e2e, test/m1-realdata-e2e, -test/load/signer) are excluded from the main workspace and carry their own -Cargo.lock. When a workspace crate such as tinycloud-core gains a dependency -or a dependency feature, the main Cargo.lock is updated by the build, but -those lockfiles are not, and `cargo ... --locked` then fails in them. CI -cannot run a `--locked` resolve of w5 (it depends on a private repository), -so this check needs no network. For every in-repo crate a separate lockfile -records (per `cargo metadata --no-deps` of the main workspace): -- each normal, non-optional dependency must appear in its lockfile entry; -- each feature it requests on a dependency that, in the main Cargo.lock, - pulls in a package (e.g. `aes` feature `zeroize`) must pull in that package - in the separate lockfile too. -Workspaces that CI can resolve are also checked with `cargo metadata ---locked` in the `lockfiles` job. - -Usage: scripts/ci/check-separate-lockfiles.py (from the repository root) -""" -import json -import pathlib -import re -import subprocess -import sys - -ROOT = pathlib.Path(__file__).resolve().parents[2] -SKIP_DIRS = {"target", "node_modules", ".git"} - - -def lock_entries(path): - """Yield (name, has_source, dependency names) for each [[package]].""" - for block in path.read_text().split("[[package]]")[1:]: - name = re.search(r'^name = "(.*)"$', block, re.M).group(1) - has_source = re.search(r"^source = ", block, re.M) is not None - deps = [] - if "dependencies = [" in block: - body = block.split("dependencies = [", 1)[1].split("]", 1)[0] - deps = re.findall(r'"([^" ]+)', body) - yield name, has_source, deps - - -def path_packages(path): - """In-repo (source-less) package name -> its dependency names.""" - return {n: set(d) for n, has_source, d in lock_entries(path) if not has_source} - - -def all_packages(path): - """Package name -> dependency names, merged over every entry of that name.""" - merged = {} - for name, _, deps in lock_entries(path): - merged.setdefault(name, set()).update(deps) - return merged - - -def workspace(): - """In-repo crates (name -> (normal deps, {dep: features})) and member dirs.""" - meta = json.loads( - subprocess.check_output( - ["cargo", "metadata", "--no-deps", "--offline", "--format-version", "1"], - cwd=ROOT, - ) - ) - crates = {} - for pkg in meta["packages"]: - normal = [d for d in pkg["dependencies"] if d["kind"] is None] - crates[pkg["name"]] = ( - sorted({d["name"] for d in normal if not d["optional"]}), - {d["name"]: d["features"] for d in normal if d["features"]}, - ) - members = { - pathlib.Path(pkg["manifest_path"]).parent - for pkg in meta["packages"] - if pkg["id"] in meta["workspace_members"] - } - return crates, members - - -def separate_lockfiles(members): - for lock in sorted(ROOT.rglob("Cargo.lock")): - rel = lock.relative_to(ROOT) - if rel == pathlib.Path("Cargo.lock") or SKIP_DIRS & set(rel.parts): - continue - if lock.parent in members: - # A main-workspace member's leftover lockfile: cargo never reads it. - continue - yield lock - - -def main(): - crates, members = workspace() - root_all = all_packages(ROOT / "Cargo.lock") - failures, checked = [], 0 - for lock in separate_lockfiles(members): - rel = lock.relative_to(ROOT) - recorded = path_packages(lock) - everything = all_packages(lock) - for name, (needed, features) in crates.items(): - if name not in recorded: - continue - checked += 1 - missing = [d for d in needed if d not in recorded[name]] - if missing: - failures.append(f"{rel}: {name} is missing {', '.join(missing)}") - for dep, wanted in features.items(): - for feature in wanted: - if feature in root_all.get(dep, ()) and feature not in everything.get(dep, ()): - failures.append( - f"{rel}: {dep} lacks {feature} (feature `{feature}` requested by {name})" - ) - for f in failures: - print(f"stale lockfile: {f}", file=sys.stderr) - if failures: - print( - "Refresh each listed lockfile, e.g. " - "`cargo update -p --manifest-path /Cargo.toml`.", - file=sys.stderr, - ) - return 1 - print(f"separate lockfiles current ({checked} in-repo crate entries checked)") - return 0 - - -if __name__ == "__main__": - sys.exit(main())