From 1e4d251ce95bb052fc8be9a31d2310b436efab4d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 8 Jul 2026 08:04:04 +0000 Subject: [PATCH] build(grammars): pin npm installs in grammar-regen scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch generate-grammar.sh to `npm ci --include=dev` and generate-mozcpp.sh's upstream install to `npm ci`, and replace the `npm install --no-save tree-sitter-c@0.23.1` override with a registry tarball fetched by exact version and verified against a recorded sha512 before extraction — no npm version resolution, and the package's install scripts are never run. Commit package-lock.json for all four internal grammar crates (previously listed in each crate's .gitignore — no lockfile was ever actually in git, contrary to the issue's premise) so `npm ci` works from a clean checkout; the mozjs lockfile on disk had drifted to tree-sitter-cli 0.25.10 against the 0.26.9 package.json pin. The crates' Cargo.toml include-lists keep the lockfiles out of the published packages. Give generate-mozjs.sh the same `set -euo pipefail` fail-loud guard and stale-clone cleanup its mozcpp sibling already had, so an aborted regen can no longer fall through to cleanup and exit 0. Both mozjs and mozcpp regens verified byte-reproducible end-to-end from a clean checkout (OpenSSF Scorecard Pinned-Dependencies, code-scanning alerts #759-#761). Fixes #1012 --- CHANGELOG.md | 19 +++++++++ .../src/developers/update-grammars.md | 7 +++- generate-grammars/generate-grammar.sh | 10 ++++- generate-grammars/generate-mozcpp.sh | 31 ++++++++++---- generate-grammars/generate-mozjs.sh | 13 +++++- tree-sitter-ccomment/.gitignore | 1 - tree-sitter-ccomment/package-lock.json | 40 +++++++++++++++++++ tree-sitter-mozcpp/.gitignore | 1 - tree-sitter-mozcpp/package-lock.json | 40 +++++++++++++++++++ tree-sitter-mozjs/.gitignore | 1 - tree-sitter-mozjs/package-lock.json | 40 +++++++++++++++++++ tree-sitter-preproc/.gitignore | 1 - tree-sitter-preproc/package-lock.json | 40 +++++++++++++++++++ 13 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 tree-sitter-ccomment/package-lock.json create mode 100644 tree-sitter-mozcpp/package-lock.json create mode 100644 tree-sitter-mozjs/package-lock.json create mode 100644 tree-sitter-preproc/package-lock.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c88e3cb6..e62f7fd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,25 @@ for historical reference. `dev` extra and `maturin` the `examples` extra so the exports cover exactly what each CI job needs. +- The grammar-regeneration scripts now install npm dependencies + hash-verified (OpenSSF Scorecard Pinned-Dependencies, code-scanning + alerts #759–#761, issue #1012). The four internal grammar crates + (`tree-sitter-ccomment`, `tree-sitter-preproc`, `tree-sitter-mozcpp`, + `tree-sitter-mozjs`) now commit their `package-lock.json` (previously + gitignored — the lockfiles were never actually in git) and + `generate-grammars/generate-grammar.sh` installs with + `npm ci --include=dev`, which fails loudly on a missing or drifted + lockfile. `generate-mozcpp.sh` uses `npm ci` inside the upstream + `tree-sitter-cpp` checkout (upstream commits a lockfile at the pinned + revision) and replaces the `npm install --no-save tree-sitter-c@0.23.1` + override with a registry tarball fetched by exact version and verified + against a recorded sha512 before extraction — no npm version + resolution at all, and the package's install scripts are never run. + `generate-mozjs.sh` gains the `set -euo pipefail` fail-loud guard its + mozcpp sibling already had, so an aborted regen can no longer fall + through to cleanup and report success. Both regens were verified + byte-reproducible from a clean checkout. + - Cleared three RUSTSEC advisories flagged by the `cargo-deny` gate. `crossbeam-epoch` (a shipped transitive dependency via `crossbeam`) moves `0.9.18` → `0.9.20` for the invalid-pointer-dereference in its diff --git a/big-code-analysis-book/src/developers/update-grammars.md b/big-code-analysis-book/src/developers/update-grammars.md index 83308efd..7dc0bf8d 100644 --- a/big-code-analysis-book/src/developers/update-grammars.md +++ b/big-code-analysis-book/src/developers/update-grammars.md @@ -46,7 +46,12 @@ Commit your changes and create a new pull request ## Internal grammars Update the version of `tree-sitter-cli` in the `package.json` file of -the internal grammar and then install the updated version. +the internal grammar, then refresh its committed `package-lock.json` +with `npm install --package-lock-only --ignore-scripts` in the same +directory and commit both files together. The regen scripts install +with `npm ci`, which fails loudly when the lockfile is missing or out +of sync with `package.json` — this keeps every regen hash-verified +and byte-reproducible (OpenSSF Scorecard Pinned-Dependencies). The five vendored grammars publish under the `bca-tree-sitter-*` namespace (see `RELEASING.md` for the rename rationale), but consumer diff --git a/generate-grammars/generate-grammar.sh b/generate-grammars/generate-grammar.sh index fa7e3035..7e1ebff7 100755 --- a/generate-grammars/generate-grammar.sh +++ b/generate-grammars/generate-grammar.sh @@ -17,8 +17,14 @@ set -euo pipefail # Enter grammar directory pushd "$1" || exit -# Install dependencies -npm install --include=dev +# Install dependencies exactly as recorded in the committed +# package-lock.json (hash-verified by npm). `npm ci` fails loudly when +# the lockfile is missing or out of sync with package.json — the right +# behaviour for a reproducibility-sensitive regen path (issue #1012, +# OpenSSF Scorecard Pinned-Dependencies). After bumping a dependency +# in package.json, refresh the lockfile with +# `npm install --package-lock-only --ignore-scripts` and commit it. +npm ci --include=dev # Generate grammar ./node_modules/.bin/tree-sitter generate diff --git a/generate-grammars/generate-mozcpp.sh b/generate-grammars/generate-mozcpp.sh index 3af66bf4..9eb02c08 100755 --- a/generate-grammars/generate-mozcpp.sh +++ b/generate-grammars/generate-mozcpp.sh @@ -15,8 +15,8 @@ # (cargo compiles against the local glibc, sidestepping the issue). # Fail loud rather than limp on with a half-regenerated, garbage -# parser: a failed download / npm install / fetch must abort the -# run, not silently fall through to `tree-sitter generate`. +# parser: a failed download / npm ci / fetch / hash check must abort +# the run, not silently fall through to `tree-sitter generate`. set -euo pipefail # Name of the tree-sitter-cpp crate @@ -70,18 +70,35 @@ git remote add origin https://github.com/tree-sitter/tree-sitter-cpp.git git fetch --depth 1 origin "$TS_CPP_SHA1" git checkout FETCH_HEAD -# Install tree-sitter-cpp dependencies -npm install -y +# Install tree-sitter-cpp dependencies exactly as recorded in the +# package-lock.json committed by upstream at the pinned revision +# (issue #1012, OpenSSF Scorecard Pinned-Dependencies). +npm ci # Pin the tree-sitter-c base grammar that tree-sitter-cpp's grammar.js # extends (`require('tree-sitter-c/grammar')`). tree-sitter-cpp declares -# it as a floating `^0.23.1`, so a bare install would silently float to +# it as a floating `^0.23.1`, so npm resolution would silently float to # the latest 0.23.x and change the generated parser — the second # non-reproducible axis behind issue #406 (the first being the # tree-sitter-cli version, pinned in tree-sitter-mozcpp/package.json). # 0.23.1 is the version the committed grammar.json/node-types.json -# correspond to; pinning it keeps the regen byte-reproducible. -npm install --no-save tree-sitter-c@0.23.1 +# correspond to. The tarball is fetched by exact version and verified +# against the sha512 recorded below (matching the npm registry's +# integrity value for tree-sitter-c@0.23.1) before it replaces +# whatever upstream's lockfile resolved — no npm resolution at all +# (issue #1012). Only grammar.js is consumed from the package, so its +# install scripts are never run. +TS_C_VERSION="0.23.1" +TS_C_TARBALL="tree-sitter-c-$TS_C_VERSION.tgz" +TS_C_SHA512="701344cff2c5d027483c3a848808059c7f0265367f08b81a395cc6e66e60d415c3eb0d9e08080fbdad40d92ce44cae60be73dd12281c41bce97d1c27b6b6eb71" +wget --header="User-Agent: big-code-analysis grammar regen" \ + -O "$TS_C_TARBALL" \ + "https://registry.npmjs.org/tree-sitter-c/-/tree-sitter-c-$TS_C_VERSION.tgz" +echo "$TS_C_SHA512 $TS_C_TARBALL" | sha512sum --check --quiet - +rm -rf node_modules/tree-sitter-c +mkdir -p node_modules/tree-sitter-c +tar -xzf "$TS_C_TARBALL" --strip-components=1 -C node_modules/tree-sitter-c +rm -f "$TS_C_TARBALL" # Exit tree-sitter-cpp directory popd || exit diff --git a/generate-grammars/generate-mozjs.sh b/generate-grammars/generate-mozjs.sh index 545edb7d..1e7327c5 100755 --- a/generate-grammars/generate-mozjs.sh +++ b/generate-grammars/generate-mozjs.sh @@ -4,6 +4,12 @@ # # Usage: ./generate-grammars/generate-mozjs.sh +# Fail loud rather than limp on with a half-regenerated, garbage +# parser: a failed download / fetch — or generate-grammar.sh aborting +# on `npm ci` lockfile drift — must abort the run, not fall through +# to the final cleanup and report success (issue #1012). +set -euo pipefail + # Name of the tree-sitter-javascript crate TS_JS_CRATE="tree-sitter-javascript" @@ -38,7 +44,12 @@ rm -rf "$CRATE_OUTPUT" "$CRATE_DIR" # Enter the mozjs directory pushd tree-sitter-mozjs || exit -# Create tree-sitter-javascript directory +# Create a fresh tree-sitter-javascript directory. Remove any leftover +# from an interrupted prior run first: under `set -e` a stale clone +# would wedge the next run at `git remote add origin` ("remote origin +# already exists", exit 3). Starting clean keeps the script idempotent +# and avoids stale partial-clone state. +rm -rf "$TS_JS_CRATE" mkdir -p "$TS_JS_CRATE" # Enter tree-sitter-javascript directory diff --git a/tree-sitter-ccomment/.gitignore b/tree-sitter-ccomment/.gitignore index 38b6f5aa..0667dcc3 100644 --- a/tree-sitter-ccomment/.gitignore +++ b/tree-sitter-ccomment/.gitignore @@ -1,4 +1,3 @@ node_modules build -package-lock.json /target/ diff --git a/tree-sitter-ccomment/package-lock.json b/tree-sitter-ccomment/package-lock.json new file mode 100644 index 00000000..7871588e --- /dev/null +++ b/tree-sitter-ccomment/package-lock.json @@ -0,0 +1,40 @@ +{ + "name": "tree-sitter-ccomment", + "version": "0.20.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-ccomment", + "version": "0.20.0", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "nan": "^2.14.2" + }, + "devDependencies": { + "tree-sitter-cli": "0.26.9" + } + }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "license": "MIT" + }, + "node_modules/tree-sitter-cli": { + "version": "0.26.9", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.26.9.tgz", + "integrity": "sha512-7l+U1RmazPVe+yA/JiX80GFOILnL/j24GbawamIzNQC8UlINrcyECbaWGaG1wuq4j/m0DQTx7Uu4r0iW9Ao1BQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" + } + } + } +} diff --git a/tree-sitter-mozcpp/.gitignore b/tree-sitter-mozcpp/.gitignore index 38b6f5aa..0667dcc3 100644 --- a/tree-sitter-mozcpp/.gitignore +++ b/tree-sitter-mozcpp/.gitignore @@ -1,4 +1,3 @@ node_modules build -package-lock.json /target/ diff --git a/tree-sitter-mozcpp/package-lock.json b/tree-sitter-mozcpp/package-lock.json new file mode 100644 index 00000000..b7ac8647 --- /dev/null +++ b/tree-sitter-mozcpp/package-lock.json @@ -0,0 +1,40 @@ +{ + "name": "tree-sitter-mozcpp", + "version": "0.20.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-mozcpp", + "version": "0.20.0", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "nan": "^2.14.2" + }, + "devDependencies": { + "tree-sitter-cli": "0.26.9" + } + }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "license": "MIT" + }, + "node_modules/tree-sitter-cli": { + "version": "0.26.9", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.26.9.tgz", + "integrity": "sha512-7l+U1RmazPVe+yA/JiX80GFOILnL/j24GbawamIzNQC8UlINrcyECbaWGaG1wuq4j/m0DQTx7Uu4r0iW9Ao1BQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" + } + } + } +} diff --git a/tree-sitter-mozjs/.gitignore b/tree-sitter-mozjs/.gitignore index 38b6f5aa..0667dcc3 100644 --- a/tree-sitter-mozjs/.gitignore +++ b/tree-sitter-mozjs/.gitignore @@ -1,4 +1,3 @@ node_modules build -package-lock.json /target/ diff --git a/tree-sitter-mozjs/package-lock.json b/tree-sitter-mozjs/package-lock.json new file mode 100644 index 00000000..c85201e1 --- /dev/null +++ b/tree-sitter-mozjs/package-lock.json @@ -0,0 +1,40 @@ +{ + "name": "tree-sitter-mozjs", + "version": "0.20.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-mozjs", + "version": "0.20.0", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "nan": "^2.14.2" + }, + "devDependencies": { + "tree-sitter-cli": "0.26.9" + } + }, + "node_modules/nan": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.27.0.tgz", + "integrity": "sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==", + "license": "MIT" + }, + "node_modules/tree-sitter-cli": { + "version": "0.26.9", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.26.9.tgz", + "integrity": "sha512-7l+U1RmazPVe+yA/JiX80GFOILnL/j24GbawamIzNQC8UlINrcyECbaWGaG1wuq4j/m0DQTx7Uu4r0iW9Ao1BQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" + } + } + } +} diff --git a/tree-sitter-preproc/.gitignore b/tree-sitter-preproc/.gitignore index 38b6f5aa..0667dcc3 100644 --- a/tree-sitter-preproc/.gitignore +++ b/tree-sitter-preproc/.gitignore @@ -1,4 +1,3 @@ node_modules build -package-lock.json /target/ diff --git a/tree-sitter-preproc/package-lock.json b/tree-sitter-preproc/package-lock.json new file mode 100644 index 00000000..9ee703a1 --- /dev/null +++ b/tree-sitter-preproc/package-lock.json @@ -0,0 +1,40 @@ +{ + "name": "tree-sitter-preproc", + "version": "0.20.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-preproc", + "version": "0.20.0", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "nan": "^2.14.2" + }, + "devDependencies": { + "tree-sitter-cli": "0.26.9" + } + }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "license": "MIT" + }, + "node_modules/tree-sitter-cli": { + "version": "0.26.9", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.26.9.tgz", + "integrity": "sha512-7l+U1RmazPVe+yA/JiX80GFOILnL/j24GbawamIzNQC8UlINrcyECbaWGaG1wuq4j/m0DQTx7Uu4r0iW9Ao1BQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" + } + } + } +}