diff --git a/.github/workflows/_build-eql-docs.yml b/.github/workflows/_build-eql-docs.yml new file mode 100644 index 000000000..63e6fb4ca --- /dev/null +++ b/.github/workflows/_build-eql-docs.yml @@ -0,0 +1,95 @@ +name: "Build EQL docs (reusable)" + +# Builds the EQL docs bundle and attaches it to the release _build-eql-sql.yml +# created earlier in the same run. Called by release.yml on both paths. +# +# Ported from packages/eql/.github/workflows/_build-docs.yml — see +# _build-eql-sql.yml for the rename and the path rewrites. +# +# This pipeline has never executed here: `docs:generate:json` had no caller at +# all until this file landed. The apt install below is load-bearing — mise does +# not provide doxygen. +on: + workflow_call: + inputs: + ref: + description: "Git ref/SHA to build docs from. Empty -> default checkout." + required: false + type: string + default: "" + tag: + description: "Full release tag. Empty -> build only, no attach." + required: false + type: string + default: "" + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + MISE_VERBOSE: "1" + +defaults: + run: + working-directory: packages/eql + +permissions: + contents: write + +jobs: + publish-docs: + name: Build and publish documentation + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + cache: false # publish path; see _build-eql-sql.yml + working_directory: packages/eql + + - name: Install Doxygen + run: | + sudo apt-get update + sudo apt-get install -y doxygen + + - name: Generate documentation + env: + TAG: ${{ inputs.tag }} + run: | + mise run docs:generate + mise run docs:generate:markdown -- "${TAG}" + mise run docs:generate:json -- "${TAG}" + + - name: Package documentation + env: + TAG: ${{ inputs.tag }} + run: mise run docs:package "${TAG}" + + - name: Upload documentation artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-docs + path: | + packages/eql/release/eql-docs-*.zip + packages/eql/release/eql-docs-*.tar.gz + # `warn` is the default, and a warning here is indistinguishable from + # a successful upload on the run page. These paths are the one thing + # the subtree import could plausibly get wrong — they are workspace- + # root relative and so are NOT covered by `defaults.run.working- + # directory` — and a wrong one publishes a release with no assets. + if-no-files-found: error + + - name: Publish documentation to release + if: ${{ inputs.tag != '' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag }} + files: | + packages/eql/release/eql-docs-*.zip + packages/eql/release/eql-docs-*.tar.gz diff --git a/.github/workflows/_build-eql-sql.yml b/.github/workflows/_build-eql-sql.yml new file mode 100644 index 000000000..f27115dd1 --- /dev/null +++ b/.github/workflows/_build-eql-sql.yml @@ -0,0 +1,124 @@ +name: "Build EQL SQL (reusable)" + +# Builds the EQL SQL bundle and attaches it to a GitHub release. Called by +# release.yml on both the production and the prerelease path, so an EQL SQL +# release has one build path. +# +# Ported from packages/eql/.github/workflows/_build-sql.yml, which GitHub never +# read. Renamed on the way in (nothing binds to the filename; "SQL" names +# nothing on its own in a monorepo), and every path moved down one level. +# +# The Multitudes deploy ping upstream ended this job with was dropped: +# MULTITUDES_ACCESS_TOKEN does not exist in this repository, so the step would +# have failed on every release under its own `continue-on-error`. + +on: + workflow_call: + inputs: + ref: + description: "Git ref/SHA to build from. Empty -> default checkout." + required: false + type: string + default: "" + tag: + description: "Full release tag (e.g. eql-3.0.6). Empty -> DEV build, no attach." + required: false + type: string + default: "" + attach: + description: "Attach the built .sql artefacts to a GitHub Release." + required: false + type: boolean + default: false + target_commitish: + description: "Non-empty -> create a release at this commit; empty -> attach to the existing release named by tag." + required: false + type: string + default: "" + prerelease: + description: "Mark the created release as a prerelease (create path only)." + required: false + type: boolean + default: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + MISE_VERBOSE: "1" + +defaults: + run: + # Every `run:` here is written against the EQL root, one level down. + working-directory: packages/eql + # No `shell: bash {0}` override (upstream had one): that disables errexit. + +permissions: + contents: write + +jobs: + build: + name: Build EQL + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + # Defaults to true; forbidden on a publish path by + # scripts/lint-no-workflow-caching.mjs. Upstream had `cache: true`. + cache: false + # `defaults.run` does not reach a `uses:` step, and mise reads config + # from cwd and its parents only. + working_directory: packages/eql + + - name: Build EQL release + # Strip `eql-` so eql_v3.version() reports bare semver. An empty TAG + # falls through to tasks/build.sh's `${usage_version:-DEV}` default. + # `--version` is not one of build.sh's declared mise sources, so force + # regeneration when unchanged SQL is being stamped for a new release. + env: + TAG: ${{ inputs.tag }} + run: mise run --force build --version "${TAG#eql-}" + + # Artifact and release paths are workspace-root relative. + - name: Upload EQL artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-release + path: | + packages/eql/release/cipherstash-encrypt.sql + packages/eql/release/cipherstash-encrypt-uninstall.sql + # `warn` is the default, and a warning here is indistinguishable from + # a successful upload on the run page. These paths are the one thing + # the subtree import could plausibly get wrong — they are workspace- + # root relative and so are NOT covered by `defaults.run.working- + # directory` — and a wrong one publishes a release with no assets. + if-no-files-found: error + + - name: Attach artefacts to existing release + if: ${{ inputs.attach && inputs.target_commitish == '' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag }} + files: | + packages/eql/release/cipherstash-encrypt.sql + packages/eql/release/cipherstash-encrypt-uninstall.sql + + - name: Create release at commit + if: ${{ inputs.attach && inputs.target_commitish != '' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag }} + target_commitish: ${{ inputs.target_commitish }} + prerelease: ${{ inputs.prerelease }} + name: ${{ inputs.tag }} + body: "The standalone eql_v3 SQL surface. See packages/eql/CHANGELOG.md." + files: | + packages/eql/release/cipherstash-encrypt.sql + packages/eql/release/cipherstash-encrypt-uninstall.sql diff --git a/.github/workflows/lint-release.yml b/.github/workflows/lint-release.yml index 26c4d147b..0125be7de 100644 --- a/.github/workflows/lint-release.yml +++ b/.github/workflows/lint-release.yml @@ -10,27 +10,37 @@ name: Lint release tooling # shellcheck over every `run:` block — which is why the workflows it gates # avoid `sed`-into-`export` (SC2001, SC2155), `ls | wc -l` (SC2012), and # dollar-braces or backticks inside single-quoted `node -e` arguments (SC2016). +# +# The EQL release workflows are in scope. They came from `packages/eql/.github/`, +# where an EQL-local copy of this file linted them and ran on nothing — along +# with its two unique checks, the `prepare-bindings-assets` shellcheck and that +# wrapper's bash unit test, neither of which ran anywhere here until now. on: pull_request: - # Exactly what the job reads: the four workflows actionlint is pointed at, - # plus the config it resolves the Blacksmith label from. The filter also - # named scripts/release-gate.mjs, scripts/ffi-release-matrix.mjs, - # scripts/lint-no-workflow-caching.mjs and package.json — none of which this - # job looks at, since it deliberately does not run `test:scripts` (see the - # job comment). Editing one booted a runner and downloaded a Go binary to - # lint four unchanged files, which is how a job trains reviewers to ignore - # it. `tests.yml`'s `lint` job runs `test:scripts` unfiltered on every PR, - # so those four are already covered. + # Exactly what the job reads: the workflows actionlint is pointed at, the + # config it resolves the Blacksmith label from, and the two shell scripts + # the last two steps run. The filter also named scripts/release-gate.mjs, + # scripts/ffi-release-matrix.mjs, scripts/lint-no-workflow-caching.mjs and + # package.json — none of which this job looks at, since it deliberately does + # not run `test:scripts` (see the job comment). Editing one booted a runner + # and downloaded a Go binary to lint four unchanged files, which is how a + # job trains reviewers to ignore it. `tests.yml`'s `lint` job runs + # `test:scripts` unfiltered on every PR, so those four are already covered. # - # `lint-release-scope.test.mjs` asserts this list and the actionlint - # argument list below stay the same set. + # `lint-release-scope.test.mjs` asserts the `.github/workflows/` half of + # this list and the actionlint argument list below stay the same set. paths: - .github/workflows/release.yml - .github/workflows/_build-ffi-artifacts.yml - .github/workflows/ffi-preflight.yml + - .github/workflows/_build-eql-sql.yml + - .github/workflows/_build-eql-docs.yml + - .github/workflows/release-plz.yml + - .github/workflows/release-postgres-eql-image.yml - .github/workflows/lint-release.yml - .github/actionlint.yaml + - packages/eql/tasks/release/*.sh workflow_dispatch: {} permissions: @@ -46,15 +56,15 @@ concurrency: jobs: # No pnpm, no Node, no install: actionlint is a downloaded Go binary and - # shellcheck ships in the runner image, so this job is a checkout and one - # command. + # shellcheck ships in the runner image, so this job is a checkout and a few + # commands. # # It deliberately does NOT also run `test:scripts` or `lint:workflow-cache`. # `tests.yml`'s `lint` job runs `test:scripts` on every pull request with no # path filter, and `tests-supply-chain.yml` runs both — so a copy here would # be the second and third run of the same checks on any PR touching the # release machinery, each behind its own uncached full-workspace install. What - # this workflow uniquely has is actionlint. + # this workflow uniquely has is actionlint, and the two shell checks below. lint: name: actionlint (release workflows) runs-on: ubuntu-latest @@ -81,5 +91,21 @@ jobs: .github/workflows/release.yml \ .github/workflows/_build-ffi-artifacts.yml \ .github/workflows/ffi-preflight.yml \ + .github/workflows/_build-eql-sql.yml \ + .github/workflows/_build-eql-docs.yml \ + .github/workflows/release-plz.yml \ + .github/workflows/release-postgres-eql-image.yml \ .github/workflows/lint-release.yml + # `prepare-bindings-assets.sh` stamps the SQL bundle and rewrites the four + # release manifests during `pnpm run version`, so a shell mistake in it + # lands mid-release with every manifest already rewritten. + - name: shellcheck (EQL release wrappers) + run: | + set -euo pipefail + shellcheck \ + packages/eql/tasks/release/prepare-bindings-assets.sh \ + packages/eql/tasks/release/prepare-bindings-assets.test.sh + + - name: prepare-bindings-assets validation unit test + run: bash packages/eql/tasks/release/prepare-bindings-assets.test.sh diff --git a/.github/workflows/rebuild-docs.yml b/.github/workflows/rebuild-docs.yml deleted file mode 100644 index 858732977..000000000 --- a/.github/workflows/rebuild-docs.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Rebuild Docs - -on: - push: - tags: - - '@cipherstash/stack@*' - -jobs: - trigger-docs-rebuild: - name: Trigger Docs Rebuild - runs-on: ubuntu-latest - steps: - - name: Send webhook - env: - WEBHOOK_URL: ${{ secrets.DOCS_WEBHOOK_URL }} - run: | - curl -X POST "$WEBHOOK_URL" diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml new file mode 100644 index 000000000..f9df54819 --- /dev/null +++ b/.github/workflows/release-plz.yml @@ -0,0 +1,162 @@ +name: "Release eql-bindings (crates.io)" + +# Publishes the `eql-bindings` crate via release-plz (crates.io Trusted +# Publishing over OIDC — no CARGO_REGISTRY_TOKEN). +# +# THE FILENAME CANNOT CHANGE: crates.io binds the publisher to it. +# +# Publish-only. Changesets owns the version — the Version Packages PR bumps +# crates/eql-bindings/Cargo.toml in lockstep with the npm package via +# scripts/sync-lockstep-versions.mjs. A release-plz `release-pr` job would +# fight it, so there isn't one. The tag is `eql-bindings-v`, chosen so a +# crate release does not fire the `eql-*` SQL/docs/image automation. +# +# INERT until the Phase-5 cutover: the `release` job is gated on +# scripts/eql-pipeline-armed.mjs, which reads FROZEN_PUBLISHERS. Before the +# first crate release this repository also needs a GPG_PRIVATE_KEY secret (it +# has none) and a crates.io Trusted Publishing entry naming cipherstash/stack + +# release-plz.yml. Neither is needed for a run that publishes nothing: the +# preflight step decides that before the key is imported. + +# Read-only by default; the publishing job escalates for itself. Enforced by +# scripts/__tests__/workflow-publish-permissions.test.mjs. +permissions: + contents: read + +on: + push: + branches: + - main + # Scoped to the subtree. Upstream had no filter and did not need one: it was + # an EQL-ONLY repository, where "every push to main" and "every EQL change" + # were the same set. Here they are not, and this job holds crates.io OIDC, + # `contents: write` and the signing key — booting it on every merge to a + # monorepo is a standing credential exposure for no release. + # + # A release that moves the crate always matches: `changeset version` runs + # scripts/sync-lockstep-versions.mjs, which rewrites + # packages/eql/crates/eql-bindings/Cargo.toml and the SQL assets beside it. + # Kept in step with the other EQL filters by + # scripts/__tests__/eql-workflow-filters.test.mjs. + paths: + - "packages/eql/**" + - ".github/workflows/release-plz.yml" + workflow_dispatch: {} + +# Never cancel — a cancelled release can leave a half-published state. +concurrency: + group: release-plz + cancel-in-progress: false + +jobs: + # No registry lookup, so it cannot race release.yml's npm publish on the same + # push. See scripts/eql-pipeline-armed.mjs. + eql-armed: + name: Is the EQL release line armed? + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + armed: ${{ steps.armed.outputs.armed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + # No pnpm install: the script imports node builtins only. + - name: Read the publisher switch + id: armed + run: node scripts/eql-pipeline-armed.mjs + + release: + name: "Release" + needs: [eql-armed] + if: needs.eql-armed.outputs.armed == 'true' + # GitHub-hosted, matching this repo's other publishing jobs. + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # the eql-bindings-v tag and release + pull-requests: write # release-plz links PRs in release notes + id-token: write # crates.io Trusted Publishing + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + # No mise step (upstream had one): nothing here runs a mise task, and + # release-plz needs only cargo and git, which the runner image ships. + + - name: Verify bundled SQL matches the crate version + # The crate `include_str!`s COMMITTED SQL and release-plz publishes the + # committed tree verbatim, so a hand-pinned Cargo.toml would ship the + # DEV placeholder as its "exact SQL". Only enforced when this run would + # actually publish; a crates.io API failure fails towards enforcing. + # + # ALSO decides whether the GPG import below runs. This step used to sit + # BELOW the key import, which meant a repository with no GPG_PRIVATE_KEY + # failed here on every push to main the moment the pipeline was armed — + # including pushes that release nothing. Nothing published (the import + # fails closed), but main went permanently red with an error that reads + # like a broken release rather than a missing secret. Ordering the + # registry question first is what makes a no-op run a no-op. + id: preflight + working-directory: packages/eql + run: | + set -euo pipefail + cargo_version="$(grep -m1 '^version = ' crates/eql-bindings/Cargo.toml | cut -d'"' -f2)" + # crates.io 403s curl's default User-Agent, which would make the + # short-circuit below unreachable. + published="$(curl -fsSL --retry 3 -H "User-Agent: cipherstash-eql-release (https://github.com/cipherstash/stack)" "https://crates.io/api/v1/crates/eql-bindings/versions" | jq -r '.versions[].num' | grep -Fx "$cargo_version" || true)" + if [[ -n "$published" ]]; then + echo "eql-bindings@${cargo_version} is already on crates.io; publish no-op — skipping the bundled-SQL guard" + echo "publish_needed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Empty `published` also covers a crates.io API failure, so this is + # the fail-towards-enforcing branch: guard the SQL, import the key. + echo "publish_needed=true" >> "$GITHUB_OUTPUT" + manifest_version="$(jq -r '.eqlVersion' crates/eql-bindings/sql/release-manifest.json)" + if [[ "$manifest_version" != "$cargo_version" ]]; then + echo "::error::release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml ('$cargo_version'). Run 'pnpm run version' from the repository root." >&2 + exit 1 + fi + # THE STAMP, not the schema name. Upstream grepped for `eql_v3`, which + # appears 23,000 times in every build including a DEV one, so the + # guard could not fire. tasks/build.sh writes the version into this + # COMMENT and it is the only record of which build produced the bytes + # — the manifest is regenerated alongside them and goes on agreeing. + if ! grep -q "COMMENT ON SCHEMA eql_v3 IS '${cargo_version}'" crates/eql-bindings/sql/cipherstash-encrypt.sql; then + stamped="$(grep -o "COMMENT ON SCHEMA eql_v3 IS '[^']*'" crates/eql-bindings/sql/cipherstash-encrypt.sql | head -1)" + echo "::error::bundled SQL was not built for ${cargo_version} — it carries ${stamped:-no version stamp}. Run 'pnpm run version' from the repository root." >&2 + exit 1 + fi + + - name: Import GPG key + # release-plz signs the commit and tag it creates. When the preflight + # says the version is already published it creates neither, so there is + # nothing to sign and no reason to require the secret. + if: steps.preflight.outputs.publish_needed == 'true' + uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + git_user_signingkey: true + git_commit_gpgsign: true + git_tag_gpgsign: true + + - name: Run release-plz release + uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5 + with: + command: release + # Required here and absent upstream: a composite action's steps do not + # inherit the caller's working-directory, so without these release-plz + # reads the monorepo root, finds no `eql-bindings` workspace, and + # exits 0 having published nothing. + manifest_path: packages/eql/Cargo.toml + config: packages/eql/release-plz.toml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-postgres-eql-image.yml b/.github/workflows/release-postgres-eql-image.yml new file mode 100644 index 000000000..c4006fd27 --- /dev/null +++ b/.github/workflows/release-postgres-eql-image.yml @@ -0,0 +1,220 @@ +name: "Release Postgres + EQL image" + +# Builds ghcr.io/cipherstash/postgres-eql (Postgres with EQL preinstalled) for +# PG 14-17 and pushes it. +# +# Dispatch-only: release.yml dispatches it on production finals. Not +# `on: release`, because a release created with GITHUB_TOKEN does not trigger +# that event — whereas a GITHUB_TOKEN workflow_dispatch does. +# +# INERT until the cutover via the eql-armed job. A second gate is outside this +# file's control: the GHCR package is still linked to +# cipherstash/encrypt-query-language, so this repository has no write access to +# it until that moves. + +on: + workflow_dispatch: + inputs: + eql_version: + description: "EQL version to embed (e.g. 3.0.6). Used as the image tag and the SQL build version." + required: true + type: string + update_floating_tags: + description: "Also update :latest, :, and : floating tags. Off by default for test runs." + required: false + type: boolean + default: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + MISE_VERBOSE: "1" + IMAGE: ghcr.io/cipherstash/postgres-eql + +permissions: + contents: read + +jobs: + eql-armed: + name: Is the EQL release line armed? + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + armed: ${{ steps.armed.outputs.armed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + - name: Read the publisher switch + id: armed + run: node scripts/eql-pipeline-armed.mjs + + build-sql: + name: Build EQL SQL + needs: [eql-armed] + if: needs.eql-armed.outputs.armed == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 20 + defaults: + run: + working-directory: packages/eql + outputs: + eql_version: ${{ steps.ver.outputs.eql_version }} + # Bare semver, passed verbatim to `mise run build --version`, so + # eql_v3.version() in the image matches the SQL release byte for byte. + build_version: ${{ steps.ver.outputs.build_version }} + update_floating_tags: ${{ steps.ver.outputs.update_floating_tags }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + cache: false # pushes an image; publish path. Upstream had `true`. + working_directory: packages/eql + + - name: Compute EQL version and tag policy + id: ver + env: + INPUT_VERSION: ${{ inputs.eql_version }} + INPUT_FLOATING: ${{ inputs.update_floating_tags }} + run: | + { + echo "eql_version=${INPUT_VERSION}" + echo "build_version=${INPUT_VERSION}" + echo "update_floating_tags=${INPUT_FLOATING}" + } >> "$GITHUB_OUTPUT" + + - name: Build EQL release SQL + env: + BUILD_VERSION: ${{ steps.ver.outputs.build_version }} + run: mise run build --version "$BUILD_VERSION" + + - name: Upload EQL SQL artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-sql + # Workspace-root relative; `defaults.run` does not reach a `uses:` step. + path: packages/eql/release/cipherstash-encrypt.sql + + build-images: + name: Build postgres-eql image (PG ${{ matrix.pg_version }}) + needs: [eql-armed, build-sql] + if: needs.eql-armed.outputs.armed == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 30 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + pg_version: ["14", "15", "16", "17"] + env: + EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} + UPDATE_FLOATING: ${{ needs.build-sql.outputs.update_floating_tags }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Download EQL SQL artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: eql-sql + # The Dockerfile COPYs it from its own build context. + path: packages/eql/docker/ + + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + cache-image: false # defaults to true; GitHub Actions cache + + - name: Set up Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + cache-binary: false # defaults to true; GitHub Actions cache + + - name: Login to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image tags + id: tags + # Always the immutable :-; the floating : only on + # release or an opted-in dispatch. + env: + PG_VERSION: ${{ matrix.pg_version }} + run: | + tags="${IMAGE}:${PG_VERSION}-${EQL_VERSION}" + if [[ "${UPDATE_FLOATING}" == "true" ]]; then + tags+=$'\n'"${IMAGE}:${PG_VERSION}" + fi + { + echo 'tags<> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + # No cache-from/cache-to: `type=gha` is the GitHub Actions cache and + # this action has no boolean to disable caching. Forbidden outright by + # scripts/lint-no-workflow-caching.mjs. + with: + context: packages/eql/docker + file: packages/eql/docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + build-args: | + PG_VERSION=${{ matrix.pg_version }} + EQL_VERSION=${{ env.EQL_VERSION }} + tags: ${{ steps.tags.outputs.tags }} + + promote-latest: + name: Promote PG17 image to :latest and : + # No eql-armed condition: both its `needs` carry one, and a skipped `needs` + # skips this job. + needs: [eql-armed, build-sql, build-images] + if: needs.build-sql.outputs.update_floating_tags == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 10 + permissions: + contents: read + packages: write + env: + EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} + + steps: + - name: Set up Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + cache-binary: false + + - name: Login to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Re-tag PG17 image as :latest and : + # imagetools preserves multi-arch manifests; `docker tag` would not. + run: | + docker buildx imagetools create \ + -t "${IMAGE}:latest" \ + -t "${IMAGE}:${EQL_VERSION}" \ + "${IMAGE}:17-${EQL_VERSION}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b0c0abec..6b20d7969 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,20 @@ name: Release JS # # See https://docs.npmjs.com/trusted-publishers#supported-cicd-providers # Enforced by scripts/__tests__/workflow-publish-permissions.test.mjs. +# +# THREE RELEASE LINES, ONE FILE: the JS packages (changesets), the seven FFI +# tarballs, and the EQL line ported here from packages/eql/.github/. They share +# a file because npm trusted publishing binds to a repository AND a workflow +# filename, so every npm publish in this repository has to happen here. +# +# `workflow_dispatch` came with the EQL port (its prerelease path is dispatched +# against a release branch). A dispatch reaches every job in the file, so +# `classify` computes the mode once and the publishing jobs key on it rather +# than assuming a push to `main`. +# +# The EQL jobs are INERT: they sit behind `eql-armed` +# (scripts/eql-pipeline-armed.mjs) as well as behind `gate`. See AGENTS.md, +# "Working on EQL". permissions: contents: read @@ -22,10 +36,120 @@ on: push: branches: - main + # The EQL prerelease path: dispatched against a release branch whose HEAD is + # an explicit `chore(release): …` marker commit. A dispatch against `main` + # classifies as `production` and behaves exactly like a push — `gate` makes it + # a no-op when there is nothing unpublished. + workflow_dispatch: {} concurrency: ${{ github.workflow }}-${{ github.ref }} jobs: + # Which release shape this run is, read by every publishing job below. + # + # production — a push to `main`, or a dispatch against it. + # prerelease — a dispatch against any other branch whose HEAD subject is + # `chore(release):`. Cuts an EQL alpha/rc without merging. + # skip — everything else, including an already-published identity. + # + # Safety on the prerelease path is the marker commit, the version regex and + # the tag short-circuit — not the branch name. Exactly `chore(release):`; a + # bare `release:` must not re-trigger a publish. + classify: + name: Classify release intent + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + mode: ${{ steps.classify.outputs.mode }} + version: ${{ steps.classify.outputs.version }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + # Depth 1 (the default): this reads `git log -1` and runs on every + # push to main. Upstream used `fetch-depth: 0`. The tag check below + # asks the API, not the remote, so no pushable credential either. + persist-credentials: false + + - id: classify + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + branch="${GITHUB_REF_NAME}" + subject="$(git log -1 --format=%s)" + mode="skip" + version="" + + if [[ "$branch" == "main" ]]; then + mode="production" + else + case "$subject" in + chore\(release\):*) + mode="prerelease" + ;; + esac + fi + + if [[ "$mode" == "prerelease" ]]; then + # Two levels down: the subtree root carries no package.json. + # Upstream this was `./packages/eql/package.json`. + version="$(node -p "require('./packages/eql/packages/eql/package.json').version")" + # prepare-bindings-assets.sh enforces the same shape with the + # suffix OPTIONAL; here it is required, because a version reaching + # this branch has already been classified a prerelease and a bare + # `3.0.6` would be one wearing the wrong identity. Checked here + # because this job runs first and that script runs last. Upstream + # checked only for a hyphen, so `3.0.6-beta` classified as a + # prerelease and got a public tag and GitHub release before the npm + # job died on it. + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then + echo "::error::a prerelease release commit must pin an exact prerelease identity (X.Y.Z-(alpha|beta|rc).N) in packages/eql/packages/eql/package.json; found '${version}'" >&2 + exit 1 + fi + # Idempotency: if this identity's tag exists the release was cut, + # so a re-push of the marker must not publish it twice. + # `matching-refs`, not `git/ref` — see the note on `publish-ffi`. + tag="eql-typescript-v${version}" + at="$(gh api "repos/${REPO}/git/matching-refs/tags/${tag}" \ + --jq ".[] | select(.ref == \"refs/tags/${tag}\") | .object.sha" 2>/dev/null || true)" + if [ -n "$at" ]; then + echo "${tag} already exists; prerelease ${version} was already published — skipping" + mode="skip" + version="" + fi + fi + + echo "classified as ${mode}${version:+ at ${version}}" + { + echo "mode=$mode" + echo "version=$version" + } >> "$GITHUB_OUTPUT" + + # May this repository publish the EQL line at all? A checkout and one `node` + # call reading a Map — no registry lookup, so every EQL job in this run gets + # the same answer. See scripts/eql-pipeline-armed.mjs. + eql-armed: + name: Is the EQL release line armed? + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + armed: ${{ steps.armed.outputs.armed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + # No pnpm install: the script imports node builtins only, as `gate` does. + - name: Read the publisher switch + id: armed + run: node scripts/eql-pipeline-armed.mjs + # WHAT STILL HAS TO BE PUBLISHED, asked of the registry rather than of the # `.changeset/` directory. "No unconsumed changesets" is also true of an # ordinary docs commit and of the commit right after a release, so gating the @@ -69,16 +193,18 @@ jobs: ffi-artifacts: name: Build FFI artifacts - needs: [gate] - if: needs.gate.outputs.ffi == 'true' + needs: [classify, gate] + # `mode == 'production'` is what workflow_dispatch cost: without it a + # dispatch against a feature branch could publish FFI from that branch. + if: needs.classify.outputs.mode == 'production' && needs.gate.outputs.ffi == 'true' uses: ./.github/workflows/_build-ffi-artifacts.yml with: ref: ${{ github.sha }} publish-ffi: name: Publish FFI packages - needs: [gate, ffi-artifacts] - if: needs.gate.outputs.ffi == 'true' + needs: [classify, gate, ffi-artifacts] + if: needs.classify.outputs.mode == 'production' && needs.gate.outputs.ffi == 'true' # GitHub-hosted for the same reason the release job is: npm rejects # provenance from a self-hosted runner with E422. This job only uploads # prebuilt tarballs, so it needs no toolchain beyond node and npm. @@ -216,7 +342,7 @@ jobs: release: name: Release - needs: [gate, publish-ffi] + needs: [classify, gate, publish-ffi] # `always()` because `publish-ffi` is SKIPPED for an ordinary JS release, # and a skipped dependency would otherwise skip this job too. # @@ -229,6 +355,7 @@ jobs: # SUCCEEDED. if: >- always() && + needs.classify.outputs.mode == 'production' && needs.gate.result == 'success' && ( needs.gate.outputs.ffi != 'true' || @@ -242,6 +369,12 @@ jobs: id-token: write # npm OIDC trusted publishing contents: write # changesets commits and pushes the Version Packages branch pull-requests: write # …and opens/updates the PR for it + # `published` alone is not enough: this job publishes every unpublished JS + # package, and a `@cipherstash/stack` release must not fire an EQL one. + outputs: + eql_published: ${{ steps.eql.outputs.eql_published }} + eql_version: ${{ steps.eql.outputs.eql_version }} + eql_prerelease: ${{ steps.eql.outputs.eql_prerelease }} steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -367,3 +500,358 @@ jobs: # the single go-live switch — set it with: # gh variable set STASH_POSTHOG_KEY --repo cipherstash/stack --body '' STASH_POSTHOG_KEY: ${{ vars.STASH_POSTHOG_KEY }} + + # Read from `publishedPackages` rather than from the tree, so the SQL + # release, docs bundle and image tag agree with what reached npm. A re-run + # against an already-published version finds nothing and the EQL branch + # skips, which is correct. + - name: Resolve the published EQL version + id: eql + if: steps.changesets.outputs.published == 'true' + env: + PUBLISHED: ${{ steps.changesets.outputs.publishedPackages }} + run: | + set -euo pipefail + version="$(node -e "const p=JSON.parse(process.env.PUBLISHED);const e=p.find(x=>x.name==='@cipherstash/eql');console.log(e ? e.version : '')")" + if [ -z "$version" ]; then + echo "@cipherstash/eql was not part of this release" + exit 0 + fi + if [[ "$version" == *-* ]]; then + prerelease=true + else + prerelease=false + fi + echo "@cipherstash/eql@${version} published (prerelease=${prerelease})" + { + echo "eql_published=true" + echo "eql_version=${version}" + echo "eql_prerelease=${prerelease}" + } >> "$GITHUB_OUTPUT" + + # ---- The EQL release line: production ------------------------------------ + # + # EQL ships as five artefacts at one version. Changesets publishes the npm + # package (above) and release-plz.yml the crate on the same push; the rest are + # built here, in the run that published, so they cannot drift from it. + # + # The `needs:` chain is load-bearing: `eql-docs` attaches to the release + # `eql-sql` creates, and `eql-image` dispatches against the tag it produced. + + eql-sql: + name: Build and attach the EQL SQL release + needs: [classify, gate, eql-armed, release] + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' + permissions: + contents: write # creates the eql- tag and release + uses: ./.github/workflows/_build-eql-sql.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.release.outputs.eql_version }} + attach: true + target_commitish: ${{ github.sha }} + prerelease: ${{ needs.release.outputs.eql_prerelease == 'true' }} + + eql-docs: + name: Build and attach the EQL docs bundle + needs: [classify, gate, eql-armed, release, eql-sql] + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' + permissions: + contents: write # attaches to the release eql-sql just created + uses: ./.github/workflows/_build-eql-docs.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.release.outputs.eql_version }} + + eql-image: + name: Dispatch the Postgres + EQL image build + needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] + # Production finals only: the floating :latest / : tags must not + # move for a prerelease. An alpha image is still buildable on demand. + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' && + needs.release.outputs.eql_prerelease == 'false' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write # gh workflow run + steps: + - name: Dispatch release-postgres-eql-image.yml + # Dispatched, not `on: release`: a GITHUB_TOKEN-created release does not + # trigger that event, but a GITHUB_TOKEN dispatch does. Against the + # eql- TAG that `eql-sql` created, so the image is built from + # the released source even if main has advanced. + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.release.outputs.eql_version }} + run: | + set -euo pipefail + # `--repo` is required: no checkout, so gh cannot infer it. + gh workflow run release-postgres-eql-image.yml \ + --repo "${GITHUB_REPOSITORY}" \ + --ref "eql-${VERSION}" \ + -f eql_version="$VERSION" \ + -f update_floating_tags=true + + # ---- The EQL release line: prerelease ------------------------------------ + # + # Dispatched against a branch whose HEAD is a `chore(release):` marker, so an + # alpha/rc can be cut without merging to `main`. Changesets is not involved — + # the version is already pinned in the manifest. + # + # SQL and docs build first and the npm publish `needs:` both: the tarball + # embeds the SQL assets, so a build failure must not arrive after the publish. + + prerelease-eql-sql: + name: Build and attach the prerelease EQL SQL + needs: [classify, gate, eql-armed] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + permissions: + contents: write + uses: ./.github/workflows/_build-eql-sql.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.classify.outputs.version }} + attach: true + target_commitish: ${{ github.sha }} + prerelease: true + + prerelease-eql-docs: + name: Build and attach the prerelease EQL docs + needs: [classify, gate, eql-armed, prerelease-eql-sql] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + permissions: + contents: write + uses: ./.github/workflows/_build-eql-docs.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.classify.outputs.version }} + + prerelease-eql-npm: + name: Publish the prerelease npm package + needs: [classify, gate, eql-armed, prerelease-eql-sql, prerelease-eql-docs] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + # GitHub-hosted for the same reason as `release` and `publish-ffi`: npm + # rejects provenance from a self-hosted runner with E422. + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + id-token: write # npm OIDC trusted publishing + contents: write # creates the eql-typescript-v tag + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + name: Install pnpm + with: + run_install: false + # Never cache the pnpm store in a credential-bearing workflow. + cache: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + # `add_shims_to_path: false` and `env: false` are load-bearing — see the + # `release` job above for both. + - name: Install mise (release:prepare_bindings_assets runs through it) + uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + cache: false + working_directory: packages/eql + add_shims_to_path: false + env: false + + # AFTER mise-action, deliberately: see the `release` job. + - name: Upgrade npm for OIDC trusted publishing + run: npm install -g npm@^11.5.1 + + # node-pty's install hook falls back to `node-gyp rebuild` on Linux and + # pnpm/action-setup v6 does not ship node-gyp. Upstream had no node-pty, + # so no such step. Asserted by workflow-node-gyp.test.mjs. + - name: Install node-gyp + run: npm install -g node-gyp + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Verify the prerelease marker and version + env: + VERSION: ${{ needs.classify.outputs.version }} + run: | + set -euo pipefail + actual="$(node -p "require('./packages/eql/packages/eql/package.json').version")" + test "$actual" = "$VERSION" || { + echo "::error::package version ${actual} does not match the prerelease identity ${VERSION}" >&2 + exit 1 + } + + - name: Prepare the exact SQL assets + # The task passes `--force` itself: `mise run build --version` does not + # treat `--version` as a cache-key input. See AGENTS.md. + working-directory: packages/eql + env: + VERSION: ${{ needs.classify.outputs.version }} + run: mise run release:prepare_bindings_assets --version "$VERSION" + + # Through turbo, not `pnpm --filter … build` as upstream ran: + # `@cipherstash/eql#build` declares `dependsOn: ["^build"]`. Asserted by + # workflow-turbo-build-deps.test.mjs. + - name: Build the package + run: pnpm exec turbo run build --filter @cipherstash/eql + + - name: Publish the package + # Two levels down — the npm package, not the subtree root. + working-directory: packages/eql/packages/eql + env: + VERSION: ${{ needs.classify.outputs.version }} + run: | + set -euo pipefail + if [ -n "$(npm view "@cipherstash/eql@${VERSION}" version 2>/dev/null)" ]; then + echo "@cipherstash/eql@${VERSION} is already published; skipping publish" + else + node scripts/npm-publish.mjs + fi + + - name: Tag the TypeScript release + env: + VERSION: ${{ needs.classify.outputs.version }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # Through the REST API, not git: the checkout has + # persist-credentials: false. `classify` reads this same tag. + tag="eql-typescript-v${VERSION}" + at="$(gh api "repos/${REPO}/git/matching-refs/tags/${tag}" \ + --jq ".[] | select(.ref == \"refs/tags/${tag}\") | .object.sha" 2>/dev/null || true)" + if [ -n "$at" ]; then + echo "tag ${tag} already exists at ${at} — skipping" + else + gh api -X POST "repos/${REPO}/git/refs" \ + -f ref="refs/tags/${tag}" -f sha="$GITHUB_SHA" >/dev/null + echo "created ${tag} at ${GITHUB_SHA}" + fi + + prerelease-eql-crate: + name: Dispatch the prerelease crate publish + needs: [classify, gate, eql-armed, prerelease-eql-sql, prerelease-eql-docs] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write # gh workflow run + contents: write # create/update the release/eql- branch ref + steps: + - name: Dispatch release-plz.yml at the release commit + # release-plz refuses a detached HEAD, so a tag dispatch does not work. + # Pin a release/eql- branch at this run's commit instead — a + # stable pointer per identity, force-updated on a retry of the same one. + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.classify.outputs.version }} + REPO: ${{ github.repository }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + branch="release/eql-${VERSION}" + if ! gh api -X POST "repos/${REPO}/git/refs" \ + -f ref="refs/heads/${branch}" -f sha="$SHA" >/dev/null 2>&1; then + gh api -X PATCH "repos/${REPO}/git/refs/heads/${branch}" \ + -f sha="$SHA" -F force=true >/dev/null + fi + # `--repo` is required: no checkout. + gh workflow run release-plz.yml --repo "$REPO" --ref "$branch" + + # What this run actually did. Worth a job because both EQL paths can + # legitimately do nothing — an already-tagged identity classifies as `skip`, + # and the run page renders that identically to "a dependency failed". + eql-summary: + name: EQL release summary + needs: + - classify + - eql-armed + # `gate` and `release` are not EQL jobs, and they are the two that most + # often decide an EQL run does nothing. `gate` exits non-zero for a frozen + # publisher — which is the NORMAL inert state — and skips `release`, whose + # `eql_published` output then gates the production EQL chain. Without them + # here every job below reads `skipped` and the table cannot separate + # "correctly inert" from "the gate refused this release" from "changesets + # failed", which is the distinction this job exists to draw. + - gate + - release + - eql-sql + - eql-docs + - eql-image + - prerelease-eql-sql + - prerelease-eql-docs + - prerelease-eql-npm + - prerelease-eql-crate + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Emit the run summary + env: + MODE: ${{ needs.classify.outputs.mode }} + VERSION: ${{ needs.classify.outputs.version }} + ARMED: ${{ needs.eql-armed.outputs.armed }} + GATE: ${{ needs.gate.result }} + RELEASE: ${{ needs.release.result }} + EQL_SQL: ${{ needs.eql-sql.result }} + EQL_DOCS: ${{ needs.eql-docs.result }} + EQL_IMAGE: ${{ needs.eql-image.result }} + PRE_SQL: ${{ needs.prerelease-eql-sql.result }} + PRE_DOCS: ${{ needs.prerelease-eql-docs.result }} + PRE_NPM: ${{ needs.prerelease-eql-npm.result }} + PRE_CRATE: ${{ needs.prerelease-eql-crate.result }} + run: | + set -euo pipefail + { + echo "## EQL release" + echo "" + echo "- mode: \`${MODE}\`" + echo "- pipeline armed: \`${ARMED}\`" + echo "- classified version: \`${VERSION:-n/a}\`" + echo "" + echo "| job | result |" + echo "| --- | --- |" + echo "| gate | ${GATE} |" + echo "| release (changesets) | ${RELEASE} |" + echo "| eql-sql | ${EQL_SQL} |" + echo "| eql-docs | ${EQL_DOCS} |" + echo "| eql-image | ${EQL_IMAGE} |" + echo "| prerelease-eql-sql | ${PRE_SQL} |" + echo "| prerelease-eql-docs | ${PRE_DOCS} |" + echo "| prerelease-eql-npm | ${PRE_NPM} |" + echo "| prerelease-eql-crate | ${PRE_CRATE} |" + if [[ "${ARMED}" != "true" ]]; then + echo "" + echo "The EQL pipeline is INERT: \`@cipherstash/eql\` is a frozen publisher in \`scripts/release-gate.mjs\`, so every job above is expected to be \`skipped\`. Deleting that entry is what arms it — see the Phase 5 cutover in \`docs/plans/2026-08-13-eql-monorepo-absorption.md\`." + elif [[ "${GATE}" != "success" ]]; then + echo "" + echo "The pipeline is armed but \`gate\` did not succeed, so nothing downstream of it ran. Run \`node scripts/release-gate.mjs\` for what it is refusing." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index 4f5bc9482..fd1555778 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -289,23 +289,15 @@ monorepo, which is where the silent failures are. because two task scripts use `git rev-parse --show-toplevel`, which after the import returns the **monorepo** root — and one of them, `tasks/test/doc-anchors.sh`, fails silently when that is wrong. -- **`packages/eql/.github/` is a dead deposit.** GitHub reads workflows from the - repo root and nowhere else, so the eleven files there run on nothing. Seven - are workflows — four publish something (`release.yml`, `release-plz.yml`, - `release-postgres-eql-image.yml`, `rebuild-docs.yml`) and three exist only to - serve them (two `workflow_call` reusables and `lint-release.yml`) — alongside - a `workflows/README.md`, the release-notes config, and two - repository-settings files. `scripts/__tests__/eql-suite-ci.test.mjs` holds - them as a **shrinking allowlist** (`UNPORTED_DEPOSIT`), asserted by equality so - it fails in both directions: porting a workflow means deleting it from the - deposit *and* from the list in the same commit, and dropping a new file in - there without listing it fails too. The same test asserts that the three SQLx - suite tasks are invoked by name from a root workflow, that every - `dorny/paths-filter` path is scoped to `packages/eql/`, and that every mise - task shelling out to cargo is either reachable from a root workflow or - exempted with a written reason. It is the guard against the failure this - absorption keeps rediscovering: a check that arrives as a file and executes on - no event reads exactly like a check that passes. +- **`packages/eql/.github/` is gone, and a test fails if it returns.** It was a + dead deposit: GitHub reads workflows from the repo root and nowhere else, so + the eleven files that arrived with the subtree ran on nothing. + `scripts/__tests__/eql-suite-ci.test.mjs` held them as a shrinking allowlist + until the release port emptied it, and now asserts the directory is gone. + **Never put a workflow under a package.** That same test also asserts that the three SQLx suite tasks are + invoked by name from a root workflow, that every `dorny/paths-filter` path is + scoped to `packages/eql/`, and that every mise task shelling out to cargo is + either reachable from a root workflow or exempted with a written reason. That scan now follows a task's `tasks/*.sh` delegations **transitively** (cycle-guarded, and it throws rather than truncating past `MAX_SCRIPT_DEPTH`), and it reads the `run:` bodies of composite actions a @@ -315,6 +307,66 @@ monorepo, which is where the silent failures are. and the job running it stopped counting as a Rust job for the cache check — while a mise task invoked from a composite action read as run by nobody, whose natural repair is an exemption claiming CI does not run it. +- **The EQL release pipeline is built and INERT, and the switch that arms it is + derived rather than flipped.** Five artefacts ship at one version — the npm + package, the `eql-bindings` crate, the SQL bundle, the docs bundle and the + `postgres-eql` image — and the workflows that produce them all live at the + repo root now: + + | file | what it does | + |---|---| + | `release.yml` | `classify` + the EQL production and prerelease jobs. It is one file with the JS and FFI releases because npm trusted publishing binds a package to a repository **and a workflow filename** | + | `_build-eql-sql.yml`, `_build-eql-docs.yml` | `workflow_call` reusables, called from both EQL paths so there is one build path per artefact | + | `release-plz.yml` | the crate. **Its filename cannot change** — crates.io Trusted Publishing binds to it | + | `release-postgres-eql-image.yml` | the GHCR image, dispatched by `release.yml` on production finals | + | `lint-release.yml` | merged into the root file of the same name | + | ~~`rebuild-docs.yml`~~ | **not ported.** It targeted the retired docs site through the deprecated `DOCS_WEBHOOK_URL`; versioned docs artifacts are still built by `_build-eql-docs.yml` | + + **Inertness is a derived switch, not a flag somebody flips.** The one piece + of state is `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs` — the existing + map recording "this package lives here but is published elsewhere" — and + every job that publishes an EQL artefact is gated on its answer. **Deleting + the `@cipherstash/eql` entry at the Phase-5 cutover is what arms the + pipeline**, and there is no second thing to remember, because the cutover has + to delete it anyway: the release gate blocks every release until it does. A + separate flag would have failed silently in the worst direction — an npm + package published with no SQL release, no docs and no crate. + + **Two readers, one map, and they are not the same code path.** Say which you + mean: + + - `scripts/eql-pipeline-armed.mjs` imports the map and answers "may this + repository publish EQL at all?". Ten jobs across `release.yml`, + `release-plz.yml` and `release-postgres-eql-image.yml` carry + `if: needs.eql-armed.outputs.armed == 'true'`. That covers the SQL bundle, + the docs, the crate, the image and the *prerelease* npm publish. + - The **production npm publish does not.** `release.yml`'s `release` job runs + `changeset publish` gated only on `needs.gate.result == 'success'`, and + `.changeset/config.json` has `"ignore": []`, so `@cipherstash/eql` is + publishable like any other workspace member. What holds it is + `release-gate.mjs` exiting non-zero for a frozen publisher, which fails the + `gate` job and skips `release` entirely. + + Both keyed on the same entry, so one deletion still arms all five. But + nothing asserts the two agree: relax the gate and the npm half opens while + `eql-armed` still reports `false` and the other four jobs skip correctly — + which is exactly the outcome the derived switch exists to prevent. This + paragraph said "every one sits behind `eql-pipeline-armed.mjs`" until an + audit read the `needs:` list. Run `node scripts/release-gate.mjs` for what + the gate is doing right now rather than inferring it from here. + + **The cutover needs two more things, neither of which a workflow can assert + ahead of time:** a `GPG_PRIVATE_KEY` secret (release-plz signs its commit and + tag, and this repository has none); write access from here to the + `ghcr.io/cipherstash/postgres-eql` package, still linked to + `cipherstash/encrypt-query-language`. The old docs-site webhook is not a + cutover dependency: that site and `DOCS_WEBHOOK_URL` are retired. +- **A release workflow may not restore a cache, including a Rust one.** + `lint-no-workflow-caching.mjs` forbids a cache restore anywhere an artefact is + published; `eql-suite-ci.test.mjs` wants `Swatinem/rust-cache` on every job + that compiles Rust. Four EQL release jobs do both, so they pay a cold compile. + The exemption is derived from the linter's own target list rather than copied, + so the two cannot disagree about which jobs those are. - **The EQL path filters are three copies of one list, and the list is derived now, not remembered.** `test-eql.yml` writes it twice (an `on: push: paths:` filter deciding whether the workflow starts at all, and a @@ -440,84 +492,73 @@ monorepo, which is where the silent failures are. `CS_WORKSPACE_CRN` re-pins both. `tests/sqlx/src/selectors.rs` holds five more workspace-keyed constants with no consumers and no guard — delete or guard them before using any of them. -- **Publishing has not moved yet.** npm trusted publishing for `@cipherstash/eql` - still names `cipherstash/encrypt-query-language` — the SLSA provenance on - `@cipherstash/eql@3.0.5` records that repository and - `.github/workflows/release.yml` — and the package's own `repository` / `bugs` - fields still point there, as do the `eql-bindings` crate's. Repointing all of - it is the Phase-5 cutover, together with the nine parked workflows above. - This was protect-ffi's situation until its own cutover, with one difference: - the guard is not a changeset lint but `scripts/release-gate.mjs`, and it is - the stronger of the two. Its `FROZEN_PUBLISHERS` map lists every package that lives here but is - published from another repository, and the gate **exits non-zero** — failing - the `gate` job, which skips `release` entirely — if such a package's committed - version is missing from npm, if any published package carries a runtime - `workspace:` range that only that package could satisfy, or — the third - check, added after a review caught it by hand — if a frozen package's - in-tree artefact is **not the bytes published under the version the tree - claims**. Any one stops the Version Packages PR and the publish alike. A - changeset-side guard sees none of them — which is why the hand-applied 3.0.5 - bump needed this one. - - **That third check is the one worth understanding before you touch - `packages/eql`.** For a package this repo publishes, in-tree bytes differing - from npm is an unreleased change — every pull request. For a frozen one it is - a contradiction: the version cannot be released from here, so the tree is not - proposing those bytes, it is *asserting they are already on npm under that - number*. Nothing local can notice when that stops being true, because - `sql/release-manifest.json` is regenerated with the SQL and goes on agreeing - with it; only the registry disagrees. This branch shipped exactly that — a - `3.0.5` subtree whose install bundle hashed `7ad9c9f8…` against npm's - `accde0030…`, because upstream restored the deprecated `ste_vec_contains` - aliases in the real release — and `stash eql install` would have executed that - SQL against a customer database, which would then have carried functions the - version it reports does not define. (The CLI now refuses a bundle whose bytes - do not hash to its own release manifest — see - `packages/cli/src/installer/bundle-digest.ts`. That catches a corrupt or - tampered `node_modules`, **not** this: a frozen-package skew regenerates the - manifest alongside the SQL, so the two agree locally and only the registry - disagrees. The release gate is still the only thing that notices.) The check - `npm pack`s the frozen package - and compares the two release manifests; `FROZEN_ARTEFACT_DIGESTS` says which - artefact, keyed identically to `FROZEN_PUBLISHERS` and deleted with it at the - cutover. - **That `npm pack` + `tar` extraction was itself executed by no test until the - #915 follow-up** — every `frozenBytesSkew` unit test injected both digests, - and the end-to-end process test shimmed `npm` with a script answering `npm - view` only, so `npm pack` got a synthetic `E404` and the comparison was - skipped on every run. Driving it for real found two defects in it: `--silent` - suppressed the very error text the function classified on, and npm answers a - missing *version* of an existing package with `ETARGET`, not `E404` — so - `@cipherstash/eql@`, the exact case the function documents, became an - uncaught throw with an empty reason, raised while building the blocker array, - which meant `reportBlockers` never printed the actionable `frozen-publisher` - remedy it had already computed. +- **`FROZEN_PUBLISHERS` is what stops this repo publishing EQL, and it is a + mechanism, not a status.** `scripts/release-gate.mjs` carries a map of every + package that lives here but is published from another repository. For each, + the gate **exits non-zero** — failing the `gate` job, which skips `release` + entirely — on any of three conditions: + + 1. the package's committed version is missing from npm; + 2. any published package carries a runtime `workspace:` range that only that + package could satisfy; + 3. the frozen package's in-tree artefact is **not the bytes published under + the version the tree claims**. + + Any one stops the Version Packages PR and the publish alike. A changeset-side + guard sees none of them. + **Whether the gate is blocking anything right now is a question for the registry, not for this file: run `node scripts/release-gate.mjs` and read what it says.** `tests.yml` runs the same script at PR time so the answer arrives a - merge earlier. `@cipherstash/eql` is the map's ONLY entry: the seven - protect-ffi packages were listed there too, and the cutover that repointed - their publisher at this repo did not take them out — which left the gate - armed against the first release that cutover had just enabled. Delete the - `@cipherstash/eql` entry in the Phase-5 cutover — + merge earlier. Which registry publishes what, and whether trusted publishing + has been repointed, is likewise configuration — check the registry, do not + read it here. This bullet used to narrate that state and was wrong twice. + + **Delete the `@cipherstash/eql` entry in the Phase-5 cutover.** `scripts/__tests__/frozen-publisher-docs.test.mjs` fails until this paragraph - goes with it, and `release-gate.test.mjs` now asserts the map carries no FFI - name, so that particular mistake cannot be made twice. - **`SECURITY.md`'s "Note on publishing" is the third document that guard - holds**, added after it was found still naming the seven - `@cipherstash/protect-ffi*` packages as published from `protectjs-ffi` — a - sentence the FFI cutover made false and nothing checked, in the one file that - tells a reporter which pipeline built the artefact they are reporting on. The - guard now fails on a doc that freezes a package the map does not, so the same - drift cannot outlive the next cutover either. - Note too that 3.0.5 did *not* come from `changeset version` — - eleven unrelated changesets were pending, so the bump was entered by hand in - `packages/eql/packages/eql/CHANGELOG.md` and the parked - `rename-ste-vec-contains.md.deferred` deleted with it, precisely so the - cutover cannot apply the same bump twice. Read that CHANGELOG entry before - assuming a version's provenance. -- **Changesets for `@cipherstash/eql` go in the repo-root `.changeset/`.** - `packages/eql/.changeset/` is an empty leftover of the subtree. + goes with it, and `SECURITY.md`'s "Note on publishing" is the third document + it holds — the one file that tells a reporter which pipeline built the + artefact they are reporting on. `release-gate.test.mjs` also asserts the map + carries no FFI name: the seven protect-ffi packages were left in it after + their own cutover, which armed the gate against the first release that + cutover had just enabled. + + **Check 3 is the one worth understanding before you touch `packages/eql`.** + For a package this repo publishes, in-tree bytes differing from npm is an + unreleased change — every pull request. For a frozen one it is a + contradiction: the version cannot be released from here, so the tree is not + proposing those bytes, it is *asserting they are already on npm under that + number*. Nothing local can notice when that stops being true, because + `sql/release-manifest.json` is regenerated with the SQL and goes on agreeing + with it; only the registry disagrees. That has happened, and the SQL would + have reached a customer database through `stash eql install`, carrying + functions the version it reports does not define. + + The CLI now refuses a bundle whose bytes do not hash to its own release + manifest (`packages/cli/src/installer/bundle-digest.ts`), but that catches a + corrupt or tampered `node_modules`, **not** this — a frozen-package skew + regenerates the manifest alongside the SQL, so the two agree locally. The + release gate is still the only thing that notices. It `npm pack`s the frozen + package and compares the two release manifests; `FROZEN_ARTEFACT_DIGESTS` + says which artefact, keyed identically to `FROZEN_PUBLISHERS` and deleted + with it. + + That `npm pack` path was itself executed by **no test** until the #915 + follow-up: every unit test injected both digests and the process test shimmed + `npm` to answer `npm view` only. Driving it for real found two defects — + `--silent` suppressed the error text the function classified on, and npm + answers a missing *version* of an existing package with `ETARGET`, not + `E404`, so the exact case the function documents became an uncaught throw + that swallowed the actionable remedy. Assume a guard is untested until you + have watched it fail. + + Note also that `@cipherstash/eql@3.0.5` did not come from `changeset + version`: eleven unrelated changesets were pending, so the bump was entered + by hand in `packages/eql/packages/eql/CHANGELOG.md`. Read that CHANGELOG + entry before assuming a version's provenance. +- **Changesets for `@cipherstash/eql` go in the repo-root `.changeset/`.** The + subtree's own `.changeset/` was deleted with the import; there is no second + one to put them in by mistake. ## Agent Skills — these ship to customers diff --git a/SECURITY.md b/SECURITY.md index 08d349c34..eab646082 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,24 +27,28 @@ lockstep with `@cipherstash/eql`. It is in scope for security reports on the same terms as the npm packages above. > **Note on publishing.** `@cipherstash/eql` and the `eql-bindings` crate are -> developed here but are still *published* from -> `cipherstash/encrypt-query-language` — the npm package's `repository` / -> `bugs` fields and the crate's `repository` / `homepage` all still name it, as -> does npm trusted publishing, and repointing every one of them is Phase 5 of -> `docs/plans/2026-08-13-eql-monorepo-absorption.md`. Everything else in the -> table above — including all seven `@cipherstash/protect-ffi*` packages, whose -> own cutover has completed — is published from this repository by -> `.github/workflows/release.yml`. Source, issues, and security reports for all -> of them belong here regardless. +> developed here but are *published* from `cipherstash/encrypt-query-language` +> until the Phase 5 cutover in +> `docs/plans/2026-08-13-eql-monorepo-absorption.md` completes. Everything else +> in the table above, including all seven `@cipherstash/protect-ffi*` packages, +> is published from this repository by `.github/workflows/release.yml`. +> **Source, issues, and security reports for all of them belong here +> regardless** — that part does not depend on which pipeline built the +> artefact. > -> The provenance attestation on a release names the repository that built it, -> and is the only thing that settles the paragraph above: -> `curl -s https://registry.npmjs.org/-/npm/v1/attestations/@cipherstash%2fprotect-ffi@0.32.0` -> returns `cipherstash/stack`; the same call against `@cipherstash%2feql@3.0.5` -> returns `cipherstash/encrypt-query-language`. Check there before repeating -> either claim: this note went on naming `cipherstash/protectjs-ffi` as the -> protect-ffi publisher through `0.32.0` — the release that proved the cutover -> and disproved the sentence. +> **Do not trust this note for which repository published a given release.** +> Registry configuration changes without touching this file, and this note has +> been wrong before: it named `cipherstash/protectjs-ffi` as the protect-ffi +> publisher through `0.32.0`, the release that disproved the sentence. The +> provenance attestation is the authority, and it is per-release: +> +> ``` +> curl -s https://registry.npmjs.org/-/npm/v1/attestations/@cipherstash%2f@ +> ``` +> +> Read the `workflow.repository` and `workflow.path` it returns. That names the +> repository and workflow that actually built the artefact you are reporting +> on. > > `scripts/__tests__/frozen-publisher-docs.test.mjs` now holds this paragraph to > `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`. It fails if the note names @@ -150,7 +154,19 @@ CI. See `skills/stash-supply-chain-security/SKILL.md` for the full guide. The `release.yml` workflow publishes packages to npm using OIDC trusted publishing (`id-token: write`). There is no long-lived `NPM_TOKEN` — the workflow deliberately avoids one, and setting one would bypass trusted -publishing. +publishing. `release-plz.yml` publishes the `eql-bindings` crate to crates.io +over the same token exchange, and likewise carries no `CARGO_REGISTRY_TOKEN`. +Both bind to a *workflow filename* at the registry, so renaming either file +silently invalidates its publisher configuration. + +`scripts/__tests__/workflow-publish-permissions.test.mjs` holds the shape those +two files must keep, as two separate equalities: who may publish (`id-token: +write`, granted per job and never at workflow level, where it would be inherited +by every job in a file the registry already trusts), and who may write to the +repository at all. They are separate because a publishing workflow also contains +jobs that create a release or dispatch another workflow — holding one does not +confer the other. Both are equalities, so either addition has to be argued for +in the same diff. [GitHub Actions cache poisoning is a known attack][1] against credential-bearing workflows. The mechanism is: @@ -163,8 +179,15 @@ workflows. The mechanism is: We mitigate this by: -- Explicitly disabling all caching in `release.yml` +- Explicitly disabling all caching in every workflow that publishes an artefact + — `release.yml`, the reusables it calls (`_build-ffi-artifacts.yml`, + `_build-eql-sql.yml`, `_build-eql-docs.yml`), `release-plz.yml` and + `release-postgres-eql-image.yml` - Automated checks for disabled caching on high-risk workflows + (`scripts/lint-no-workflow-caching.mjs`), working from an allowlist of audited + actions rather than a denylist of cache actions — so an action it has never + seen is a finding by default. The cost is accepted: four EQL release jobs + compile Rust with no cache restore. [1]: https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/ diff --git a/docs/plans/2026-08-13-eql-monorepo-absorption.md b/docs/plans/2026-08-13-eql-monorepo-absorption.md index 19e30c5d1..cffeaf553 100644 --- a/docs/plans/2026-08-13-eql-monorepo-absorption.md +++ b/docs/plans/2026-08-13-eql-monorepo-absorption.md @@ -209,7 +209,7 @@ EQL ships everything at a single version V, computed by Changesets from `package | `@cipherstash/eql` (npm) | Changesets, `release.yml` | npm OIDC, bound to `release.yml` | | `eql-bindings` (crates.io) | release-plz, `release-plz.yml` | crates.io OIDC, bound to `release-plz.yml` | | SQL install/uninstall bundle | `_build-sql.yml`, attached to the `eql-` GitHub release | — | -| API docs (HTML/XML/Markdown) | `_build-docs.yml` + `rebuild-docs.yml` webhook to Vercel | `DOCS_WEBHOOK_URL` secret | +| API docs (HTML/XML/Markdown) | `_build-docs.yml`, attached to the `eql-` GitHub release | — | | `ghcr.io/cipherstash/postgres-eql` | `release-postgres-eql-image.yml` | `GITHUB_TOKEN` | plus **dbdev**, which is documented as manual and lagging. @@ -306,27 +306,30 @@ The protect-ffi lesson applies verbatim: **a check that arrives as a file and th - [x] Satisfy this repo's credentialed-job guards for the two `CS_*`-bearing jobs — `BINDING_EXEMPT_JOBS` entries with the crate-not-binding reason, a `require-cs-secrets` step before the credentialed work, and the `vars`/`secrets` split this repo uses. See the finding above; `scripts/__tests__/ffi-binding-step-order.test.mjs` fails the build until all three are done. - [x] Fork-PR-skip the credentialed jobs, matching the other `integration-*.yml` workflows. Already true upstream — but **two of this repo's guards had to be extended rather than exempted**, which the plan did not anticipate. `workflow-dispatch-job-conditions.test.mjs` compared whole conditions and required exactly one spelling; EQL's two jobs `&&` the guard with a relevance gate, so the single-spelling rule now applies to the *clause*, compared verbatim, with another conjunct permitted beside it. Its synthetic contexts gained `merge_group` and a permissive `needs` (without which the compound conditions read as skipped on every event), and the verdict table is now derived from the workflow's declared triggers — `test-eql.yml` has no `push:`, and asserting about a run that cannot happen is not a check. Its evaluator also had to model `always()` for the `ci-required` aggregator: total on every event, therefore modellable; `success()` and argument-taking calls still throw. - [x] Port `bench-eql.yml` and `macro-expand-eql.yml`. `bench` needed three things beyond paths: it holds `CS_*`, so it takes the `vars`/`secrets` split and a `require-cs-secrets` pre-flight (worth more on a scheduled 60-minute job than on a PR — nobody watches a nightly start); and its three actions were on floating major tags, now SHA-pinned to the same commits `test-eql.yml` uses. It also joins `EXPECTED_ASYMMETRIES` in the paths-filter parity guard: `push` only, no second list to drift from. -- [ ] **Moved to Phase 5** — `lint-release.yml` and the two reusable workflows, plus the `cache: false` additions and the `lint-no-workflow-caching.mjs` `TARGETS`/`AUDITED_ACTIONS` work. All of it is about the release machinery: EQL's `lint-release.yml` points actionlint at four release workflows that do not exist at the root yet, and `_build-sql.yml` / `_build-docs.yml` are `workflow_call`-only, reached from EQL's `release.yml` alone. Porting them here would land a gate aimed at absent files. The one genuinely Phase-4 piece is its `shellcheck` + `prepare-bindings-assets.test.sh` half, which covers the asset builder the root `version` script already calls — carry that into the root `lint-release.yml` rather than leaving it with the rest. +- [x] **Moved to Phase 5, and done there** — `lint-release.yml` and the two reusable workflows, plus the `cache: false` additions and the `lint-no-workflow-caching.mjs` `TARGETS`/`AUDITED_ACTIONS` work. All of it is about the release machinery: EQL's `lint-release.yml` points actionlint at four release workflows that do not exist at the root yet, and `_build-sql.yml` / `_build-docs.yml` are `workflow_call`-only, reached from EQL's `release.yml` alone. Porting them here would land a gate aimed at absent files. The one genuinely Phase-4 piece is its `shellcheck` + `prepare-bindings-assets.test.sh` half, which covers the asset builder the root `version` script already calls — carry that into the root `lint-release.yml` rather than leaving it with the rest. - [x] Split cargo off the default test path exactly as protect-ffi did. **Half of this was already true and the other half had nowhere to attach.** EQL has no npm-script layer over its cargo work — the checks are mise tasks invoked straight from workflows — so `@cipherstash/eql`'s `test` is `vitest run` and there is no `test:cargo` to be reachable from. The property therefore lands one level up, in `scripts/__tests__/eql-suite-ci.test.mjs`: every mise task that invokes cargo must be reached by a root workflow, directly or through `depends`. 19 tasks qualify (15 if you read only the task block — the rest hide behind `run = "bash tasks/test/*.sh"`, which is the class most worth checking, so the scan follows that hop and its floor is set to fail if it stops). Five are unreached and each is named with its reason: a watcher, two `:regen` write-halves whose read-halves run in CI, an unsharded local variant, and a unit-test task `test:crates` subsumes. - [x] Add the EQL Cargo workspace to Dependabot. **The existing lockfile check would not have caught this**: `supply-chain.e2e.test.ts` asserts coverage per *ecosystem*, and `cargo` was already covered by the protect-ffi entry, so `packages/eql/Cargo.lock` read as monitored while nothing proposed an update for it. Dependabot's cargo `directory:` is one workspace root, not a glob. The `ignore` list carries the same four CipherStash crates, and here the argument is sharper than analogy: `tests/sqlx/Cargo.toml` pins `cipherstash-client = "=0.42.0"`, the same exact pin at the same version as protect-ffi, so a PR moving one workspace and not the other reintroduces exactly the skew this absorption removes. - [x] `.github/actionlint.yaml` must declare the Blacksmith runner labels the imported workflows use — one label, `blacksmith-16vcpu-ubuntu-2204`, used by every job in the suite. Verified by running actionlint over the ported files: identical shellcheck findings before and after, and nothing structural. -- [ ] **Moved to Phase 5 — an ordering defect, found by executing this phase.** Four of the ten workflows *are* the release machinery, and Phase 5 ports them; deleting the directory here would mean reconstructing a publish pipeline from git history at the one moment nobody wants to be doing that. What lands now instead is the shrinking allowlist in `eql-suite-ci.test.mjs`: every file still in the deposit, named with the reason it has not been ported. It is an equality, so it fails in both directions — a file that comes back fails, and so does an entry left behind after its file is gone, which is what forces the last removal to be `rm -r` plus a straight `existsSync(...) === false`. +- [x] **Moved to Phase 5, and done there — an ordering defect, found by executing this phase.** Four of the ten workflows *are* the release machinery, and Phase 5 ports them; deleting the directory here would mean reconstructing a publish pipeline from git history at the one moment nobody wants to be doing that. What lands now instead is the shrinking allowlist in `eql-suite-ci.test.mjs`: every file still in the deposit, named with the reason it has not been ported. It is an equality, so it fails in both directions — a file that comes back fails, and so does an entry left behind after its file is gone, which is what forces the last removal to be `rm -r` plus a straight `existsSync(...) === false`. - [x] **Write the test that asserts a root workflow still runs the SQLx suite** — `scripts/__tests__/eql-suite-ci.test.mjs`, in the shape of `packages/protect-ffi/src/integrationSuiteCi.test.ts` and reusing its `executablePart` trick, so a `paths:` entry or a comment naming a task cannot satisfy it. Mutation-checked six ways across this phase: unprefixing a paths-filter entry, deleting the partition step, demoting it to a comment, pointing rust-cache back at the monorepo root, dropping the e2e pre-flight, and respelling the fork clause with its disjuncts swapped — each fails the intended guard and only that one. ## Phase 5 — release cutover Five surfaces, each flipped independently, each inert until its own cutover. -- [ ] **Carried over from Phase 4, deliberately.** Port `lint-release.yml` and the two reusable workflows (`_build-sql.yml`, `_build-docs.yml`) alongside the release workflows they gate and are called from — a lint aimed at four absent files is worse than no lint. Add `cache: false` to the three `jdx/mise-action` steps that omit it (`_build-sql.yml`, `_build-docs.yml`, `release-postgres-eql-image.yml`), and add those two plus `release-plz.yml` to `scripts/lint-no-workflow-caching.mjs`'s `TARGETS`, with their remote actions in `AUDITED_ACTIONS` — or the gate fails on the addition itself. -- [ ] **Then delete `packages/eql/.github/` entirely**, and convert the shrinking allowlist in `scripts/__tests__/eql-suite-ci.test.mjs` to a straight `existsSync(...) === false`, alongside the existing protect-ffi assertion in `lintWiring.test.ts`. The allowlist is an equality, so it will already be failing by the time the last file goes — that is what makes this step forced rather than remembered. +- [x] **Carried over from Phase 4, deliberately.** Port `lint-release.yml` and the two reusable workflows (`_build-sql.yml`, `_build-docs.yml`) alongside the release workflows they gate and are called from — a lint aimed at four absent files is worse than no lint. Add `cache: false` to the three `jdx/mise-action` steps that omit it (`_build-sql.yml`, `_build-docs.yml`, `release-postgres-eql-image.yml`), and add those two plus `release-plz.yml` to `scripts/lint-no-workflow-caching.mjs`'s `TARGETS`, with their remote actions in `AUDITED_ACTIONS` — or the gate fails on the addition itself. **Two corrections from doing it.** The three files did not omit `cache:`; they set it to `true`, one hit each, so the fix was to flip the value rather than add a key (adding one produces a duplicate YAML key). And the reusables were renamed on the way in — `_build-eql-sql.yml`, `_build-eql-docs.yml` — because "SQL" and "docs" name nothing on their own in a monorepo and nothing binds to either filename; `release-plz.yml` is the one that could not be renamed. `release-plz.yml` also lost its mise step entirely: no step in that job runs a mise task, and the publishing-job rule (`add_shims_to_path: false`) would have left mise's toolchain off PATH anyway. +- [x] **Then delete `packages/eql/.github/` entirely**, and convert the shrinking allowlist in `scripts/__tests__/eql-suite-ci.test.mjs` to a straight `existsSync(...) === false`, alongside the existing protect-ffi assertion in `lintWiring.test.ts`. The allowlist is an equality, so it will already be failing by the time the last file goes — that is what makes this step forced rather than remembered. - [x] ~~Add `scripts/lint-no-eql-changeset.mjs` in the shape of the FFI guard~~ — **superseded, and do not restore the parked changeset.** The guard landed instead as `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`, which is strictly stronger: it exits non-zero on any frozen package whose committed version is missing from npm *and* on any runtime `workspace:` range only that package could satisfy, a pair of conditions no changeset-side guard can see — which is why the hand-applied 3.0.5 bump needed this guard rather than that one. (Run `node scripts/release-gate.mjs` for what it blocks at any given moment; that answer comes from the registry and is not recorded here.) It runs at PR time via `tests.yml` and at release time via `release.yml`. `.changeset/rename-ste-vec-contains.md.deferred` was deleted deliberately in `9b1c44d9` — the 3.0.5 bump was entered by hand in the CHANGELOG, so re-parking that `major` changeset would make the cutover bump the package a second time (3.0.5 → 4.0.0) for a rename already released. **Phase-5 action: delete the `@cipherstash/eql` entry from `FROZEN_PUBLISHERS`, nothing more.** -- [ ] Repoint all EQL manifests at `cipherstash/stack`: `packages/eql/packages/eql/package.json` — `repository.url`, `repository.directory` → `packages/eql/packages/eql`, **and `bugs.url`** (line 13, still `encrypt-query-language/issues`, which would send users to an archived repo) — plus `crates/eql-bindings/Cargo.toml` (`repository`, `homepage`) and `release-plz.toml`'s prose. npm and crates.io both reject a publish whose repository field does not match; `bugs.url` is not enforced, which is why it is the one that survives a cutover unnoticed. -- [ ] Port `release.yml`'s EQL half into this repo's `release.yml` — the existing gate-then-publish structure already models "one version authority plus N idempotent publishers", which is the shape it was derived from. Preserve the `classify` prerelease path and the `eql-typescript-v` idempotency short-circuit. -- [ ] Port `release-plz.yml` unchanged in filename — crates.io trusted publishing binds to it. -- [ ] Port `release-postgres-eql-image.yml` and `rebuild-docs.yml`. +- [x] Repoint all EQL manifests at `cipherstash/stack` (`e52d3317`): `packages/eql/packages/eql/package.json` — `repository.url`, `repository.directory` → `packages/eql/packages/eql`, **and `bugs.url`** (line 13, still `encrypt-query-language/issues`, which would send users to an archived repo) — plus `crates/eql-bindings/Cargo.toml` (`repository`, `homepage`) and `release-plz.toml`'s prose. npm and crates.io both reject a publish whose repository field does not match; `bugs.url` is not enforced, which is why it is the one that survives a cutover unnoticed. +- [x] Port `release.yml`'s EQL half into this repo's `release.yml` — the existing gate-then-publish structure already models "one version authority plus N idempotent publishers", which is the shape it was derived from. Preserve the `classify` prerelease path and the `eql-typescript-v` idempotency short-circuit. **What it cost, all of it found by this repo's guards rather than by review.** `workflow_dispatch` on `release.yml` (the prerelease path is dispatched) means every job in the file is reachable by hand, so `classify` gates the FFI and changesets jobs too. Three guards had to grow: `workflow-publish-permissions.test.mjs` needed `REPO_WRITE_JOBS` beside `OIDC_JOBS`, because six new jobs need a writable scope and must NOT be able to publish; `workflow-dispatch-job-conditions.test.mjs` needed a reasoned skip list, because the production and prerelease halves are mutually exclusive by construction and no single synthetic context can run both; and `eql-suite-ci.test.mjs`'s rust-cache requirement needed a publish-path exemption derived from the caching linter's own targets. Two real defects came out of the same pass: the prerelease job needed `Install node-gyp` before `pnpm install` (node-pty, which upstream did not have) and `pnpm exec turbo run build` rather than `pnpm --filter … build` (`^build` dependencies, which upstream did not have either). +- [x] Port `release-plz.yml` unchanged in filename — crates.io trusted publishing binds to it. It also needed `manifest_path: packages/eql/Cargo.toml` and `config: packages/eql/release-plz.toml`, which upstream did not: a composite action's steps do not inherit the caller's `defaults.run.working-directory`, so a `working-directory:` on the step would have done nothing and release-plz would have read the monorepo root, found no workspace naming `eql-bindings`, and exited 0 having published nothing. +- [x] Port `release-postgres-eql-image.yml`; deliberately retire `rebuild-docs.yml`. Upstream's workflow posted to the deprecated `DOCS_WEBHOOK_URL` for an old Vercel-hosted docs site. That site is retired, so neither the EQL nor Stack release path carries the webhook forward. The versioned EQL docs bundle remains a release artifact built by `_build-eql-docs.yml`. - [ ] **Repoint trusted publishing**, only after a versioned dry run: npm for `@cipherstash/eql` (workflow `release.yml`, explicitly selecting `npm publish` under Allowed actions), and crates.io for `eql-bindings` (workflow `release-plz.yml`). -- [ ] Move the `DOCS_WEBHOOK_URL` secret and the GHCR package ownership. -- [ ] Activate the parked changesets, merge the Version Packages PR, and verify at one version V: the npm tarball, the crate, the `eql-` release with both SQL assets, the docs rebuild, and the image tags. +- [x] `GPG_PRIVATE_KEY` secret added to `cipherstash/stack` (release-plz signs its commit and tag). The workflow's GPG import is now gated on the crates.io preflight, so a run that publishes nothing does not need it. +- [x] GHCR: `cipherstash/stack` granted Write under **Manage Actions access** on `ghcr.io/cipherstash/postgres-eql`. This is an allowlist, not an exclusive binding, so the old repository can keep its access until the cutover is proven. The `org.opencontainers.image.source` label in `packages/eql/docker/Dockerfile` is what re-links the package on push and now names `cipherstash/stack` — it must merge before anything pushes. +- [x] `DOCS_WEBHOOK_URL` is deprecated with the retired docs site. Remove both the pre-existing Stack webhook workflow and the imported EQL rebuild job; it is not a cutover dependency. +- [ ] crates.io Trusted Publishing for `eql-bindings` → `cipherstash/stack` + `release-plz.yml`. **Blocked on this pipeline PR merging**: the workflow has to exist on the default branch first. +- [ ] Activate the parked changesets, merge the Version Packages PR, and verify at one version V: the npm tarball, the crate, the `eql-` release with SQL and docs assets, and the image tags. - [ ] Update the dbdev runbook for the new paths. It stays manual. ## Phase 6 — flip @@ -359,11 +362,11 @@ Five surfaces, each flipped independently, each inert until its own cutover. - [x] `cargo build -p protect-ffi` succeeds against the path-dep `eql-bindings` — host (4m 51s) and `wasm32-unknown-unknown` (1m 18s), plus `test:cargo` at 310 passed / 0 failed and `cargo fmt --check` clean - [x] The credentialed protect-ffi integration suite passes unchanged against the path dep. **Split out of the line above deliberately**: they were one box, and one box that is half-runnable is a box that gets ticked. Closed by runs [`31759054388`](https://github.com/cipherstash/stack/actions/runs/31759054388) and [`32316091388`](https://github.com/cipherstash/stack/actions/runs/32316091388) on PR #885 — the credentialed, Dockerised `integration-protect-ffi.yml`, green at both heads that carried the flip. Scope and limits are on the Phase-3 box of the same name; do not restate the verdict without them - [x] No manifest names `eql-bindings` or `@cipherstash/eql` by registry version, except `packages/protect-ffi/integration-tests` under a named exemption with a written reason — `scripts/lint-no-eql-registry-pins.mjs`, run from `tests.yml`'s `lint` job, mutation-checked four ways -- [ ] `packages/eql/.github/` is deleted and a test fails if it returns — **Phase 5**, once the release workflows are ported. Until then `eql-suite-ci.test.mjs` holds the deposit as an equality against the list of files still waiting. +- [x] `packages/eql/.github/` is deleted and a test fails if it returns. Three of the eleven files were dropped rather than ported, each with the reason recorded where the allowlist used to be: the actionlint config and the issue template are duplicates the root already carries, and `workflows/README.md` documented a merge-queue model this repository does not have. - [x] The SQLx matrix runs from a root workflow, fork-PR-skipped, credentialed, PG 14–17 in the merge queue - [x] `scripts/__tests__/ffi-binding-step-order.test.mjs` passes with the EQL jobs present — exempted from the binding build with a reason, **not** exempted from `require-cs-secrets` -- [ ] No `jdx/mise-action` step in any release workflow omits `cache:`, and the caching lint's target list covers `release-plz.yml` and the image publisher -- [x] `actionlint` clean over every ported workflow — verified for `test-eql.yml`, `bench-eql.yml` and `macro-expand-eql.yml`. "Clean" means no structural, expression or runner-label findings; the shellcheck style/info diagnostics are upstream's and are byte-for-byte the same set before and after the port, which is the comparison that shows the port introduced nothing +- [x] No `jdx/mise-action` step in any release workflow restores a cache, and the caching lint's target list covers all four ported release workflows. `AUDITED_ACTIONS` gained seven entries; two of the three Docker ones turned out to have a GitHub-Actions-cache input defaulting to **true** (`cache-image`, `cache-binary`), and `docker/build-push-action` — which has no such toggle — is held by a new `forbiddenInputs` rule on `cache-from` / `cache-to`, since `type=gha` names that cache directly +- [x] `actionlint` clean over every ported workflow — verified for `test-eql.yml`, `bench-eql.yml` and `macro-expand-eql.yml`, and now for the four release workflows too, which `lint-release.yml` lints on every PR that touches them rather than leaving it to a local run. "Clean" means no structural, expression or runner-label findings; the shellcheck style/info diagnostics are upstream's and are byte-for-byte the same set before and after the port, which is the comparison that shows the port introduced nothing - [ ] `release.yml` passes `version: pnpm run version` to `changesets/action`, and a dry run shows `Cargo.toml` and the SQL assets moving to the computed version — the hook is dead without it - [ ] All EQL manifests read `cipherstash/stack`, including `repository.directory` and `bugs.url` - [ ] `pnpm run code:check` error-free across the merged tree diff --git a/package.json b/package.json index 18b28ec98..87769b6c8 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "changeset": "changeset", "changeset:version": "pnpm run version", "changeset:publish": "pnpm run release", - "dev": "turbo dev --filter './packages/*'", + "dev": "turbo dev --filter './packages/**'", "clean": "rimraf --glob **/.next **/.turbo **/dist **/node_modules", "code:fix": "biome check --write", "code:check": "biome check", diff --git a/packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml b/packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml deleted file mode 100644 index 49af6b6fb..000000000 --- a/packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Docs feedback -description: Feedback to help make our docs more effective. -title: "[Docs]: " -labels: ["docs", "triage"] -assignees: - - kateandrews -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to give us feedback! - - type: input - id: contact - attributes: - label: Contact details - description: How can we get in touch with you if we need more info? - placeholder: you@example.com - validations: - required: false - - type: textarea - id: what-problem - attributes: - label: What problem were you trying to solve? - placeholder: Tell us what you were looking for in our docs. - validations: - required: true - - type: textarea - id: additional-info - attributes: - label: Anything else you'd like us to know? - placeholder: Let us know if you've got any additional feedback. - validations: - required: false - - type: checkboxes - id: terms - attributes: - label: Code of conduct - description: By submitting this issue, you agree to follow our [Code of conduct](https://github.com/cipherstash/encrypt-query-language/blob/main/CODE_OF_CONDUCT.md). - options: - - label: I agree to follow this project's Code of conduct - required: true diff --git a/packages/eql/.github/actionlint.yaml b/packages/eql/.github/actionlint.yaml deleted file mode 100644 index b6a2e7da2..000000000 --- a/packages/eql/.github/actionlint.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# actionlint does not know repo-specific / provider-specific runner labels. -self-hosted-runner: - labels: - - blacksmith-16vcpu-ubuntu-2204 diff --git a/packages/eql/.github/release.yml b/packages/eql/.github/release.yml deleted file mode 100644 index ec475e66b..000000000 --- a/packages/eql/.github/release.yml +++ /dev/null @@ -1,5 +0,0 @@ -changelog: - categories: - - title: Changes since the last release - labels: - - "*" diff --git a/packages/eql/.github/workflows/README.md b/packages/eql/.github/workflows/README.md deleted file mode 100644 index cbe72a1c5..000000000 --- a/packages/eql/.github/workflows/README.md +++ /dev/null @@ -1,228 +0,0 @@ -# CI workflows - -This directory holds every GitHub Actions workflow for EQL. This README is the -authoritative **inventory** (what workflows exist and when they fire) and -**coverage map** (which job runs which checks, and where each test suite -actually runs). - -- [Workflow inventory](#workflow-inventory) -- [`test-eql.yml` — the merge gate](#test-eqlyml--the-merge-gate) -- [Coverage map: job → task → what it checks](#coverage-map-job--task--what-it-checks) -- [Where each test suite runs](#where-each-test-suite-runs) -- [Known gaps](#known-gaps) - ---- - -## Workflow inventory - -| Workflow | Triggers | What it does | Gates merge? | -|---|---|---|---| -| **test-eql.yml** | `pull_request`, `merge_group`, `workflow_dispatch` | Full test/lint/validate matrix; the one required check | **Yes** — `ci-required` | -| **release-postgres-eql-image.yml** | `workflow_dispatch` (dispatched by `release.yml` on production finals) | Build & push the Postgres+EQL Docker image to GHCR | No | -| **release.yml** | `push: main`, `push: eql_v3`, `workflow_dispatch` | Unified release entrypoint: production on `main`, prerelease on `eql_v3` when the commit is an explicit `chore(release): ...` marker | No | -| **release-plz.yml** | `push: main`, `workflow_dispatch` | Publish the `eql-bindings` crate to crates.io (Trusted Publishing) + open the release PR | No | -| **bench-eql.yml** | `push: main` (paths), `schedule` 02:00 UTC daily, `workflow_dispatch` | `test:bench` (bench cargo feature). **Never runs on PRs** | No | -| **macro-expand-eql.yml** | `schedule` 03:00 UTC daily, `workflow_dispatch` | Regenerate the integer `cargo expand` matrix snapshot; needs pinned nightly | **No — explicitly non-blocking** | -| **rebuild-docs.yml** | `push: tags` | Fire a docs-site rebuild webhook | N/A | - -Only **test-eql.yml** gates merges. Bench regressions and stale `cargo expand` -snapshots surface on the nightly schedule, not on the PR that caused them. - -### Two release tag families - -There are two independent release flows keyed on distinct git tags: - -- **`eql-`** (e.g. `eql-3.0.0`) — the EQL **SQL surface** release, cut - by `release.yml` (production on `main`, prerelease on `eql_v3`), which builds - and attaches the SQL + docs in-run. On production finals `release.yml` also - dispatches `release-postgres-eql-image.yml` (the Postgres+EQL Docker image). -- **`eql-bindings-v`** (e.g. `eql-bindings-v0.1.0`) — the **`eql-bindings` - Rust crate** release, cut automatically by `release-plz.yml` when its release - PR merges. Publishes to crates.io only. - -The three SQL-surface workflows explicitly **exclude** `eql-bindings-*` tags -(`!startsWith(...'eql-bindings')` guards; a `!eql-bindings-*` filter in -`rebuild-docs.yml`) so a crate release never triggers an SQL build, Docker -image, or docs-site rebuild. `eql-bindings` is tested on every PR by the -`rust-crates` job (`mise run test:crates` plus a `cargo publish --dry-run` -packaging gate); every other workspace crate is `publish = false`. - ---- - -## `test-eql.yml` — the merge gate - -Fast PR feedback + a thorough pre-merge gate, using a merge queue and a single -aggregated required check. - -### Two run shapes - -`setup` derives the matrix from the event: - -| Event | Trigger | Matrix | Purpose | -|---|---|---|---| -| `pull_request` | push to a PR | PG17 × 4 shards | fast developer feedback | -| `merge_group` | "Merge when ready" → queued | PG14–17 × 2 shards | full pre-merge gate on the real merged state | -| `workflow_dispatch` | manual run | PG17 × 4 shards (PR shape) | ad-hoc | - -The PR run is feedback only. The merge-queue run is the gate. - -> **No `push` trigger.** Under a required merge queue, push-to-main validation is -> redundant — the queue already validated the exact merge commit, and branch -> protection blocks direct pushes. - -### Relevance skip applies to PRs only - -Each heavy job runs when: -`merge_group || workflow_dispatch || (pull_request && relevant == 'true')`. - -So the `changes` relevance filter only gates the **`pull_request`** event — a -docs-only PR skips the heavy jobs on its PR run. On `merge_group` (and -`workflow_dispatch`) every job runs **unconditionally**: a queued PR always pays -the full gate regardless of which files it touched. - -The relevance filter (`changes` job) marks a PR relevant when any of these -changed: `.github/workflows/test-eql.yml`, `src/**`, `sql/**`, `tests/**`, -`tasks/**`, `crates/**`, `Cargo.toml`, `Cargo.lock`, `mise.toml`. Note `docs/**` -is **not** in the filter — see [Known gaps](#known-gaps). - -### How the queue works - -1. Click **Merge when ready** — the PR is queued, not merged. -2. GitHub builds a temporary branch = `main` + this PR (+ any PRs ahead in the - queue) and fires `merge_group`, so CI tests the **post-merge state**. -3. The full PG14–17 × 2 matrix (plus the single-run jobs) runs and feeds - `ci-required`. -4. `ci-required` green → PR is **merged**. Red → PR is **removed from the - queue**; `main` is untouched. - -This catches semantic conflicts — two PRs that each pass alone but break -together — which PR-only checks never test. - -### The `ci-required` aggregator - -Per-event matrices make leaf job names unstable (`Shard PG17 1/4` on a PR vs. -`Shard PG14 1/2` in the queue), so leaf names can't be named as required checks. -Instead, one aggregator job (id and display name `ci-required`) `needs:` every -job, runs with `if: always()`, and passes only if each needed result is -`success` **or** `skipped`. Mark **only `ci-required`** as the required status -check. - -- `if: always()` — runs even when dependencies fail/skip, so the check always - reports (a never-reported required check leaves the queue stuck *Pending*). -- `skipped` counts as pass — a docs-only PR skips the heavy jobs on its PR run - but must still report Success so the PR stays eligible to queue. - ---- - -## Coverage map: job → task → what it checks - -All jobs run on `blacksmith-16vcpu-ubuntu-2204`. "PG set" follows the event -(PG17 on PR / dispatch, PG14–17 in the queue). - -| Job | mise task(s) | Checks | DB | `CS_*` creds | -|---|---|---|---|---| -| **changes** | — | compute relevance | no | no | -| **setup** | — | compute PG × shard matrix | no | no | -| **build-archive** | `test:sqlx:archive` | Build EQL, run prep, **generate fixtures**, compile every `tests/sqlx` binary (**default features**) into a nextest archive; upload archive + `release/*.sql` | yes (PG17) | **yes (sole holder)** | -| **test** (sharded) | `test:sqlx:partition` | Run the archived sqlx binaries (default features), hash-partitioned across shards | yes (per PG) | no (replays archive) | -| **e2e** | `test:sqlx:e2e` | The `proptest-e2e` fresh-encryption suites (`e2e_oracle` **and** `float_special`) — PG17 only, version-independent | yes (PG17) | **yes** | -| **known-failures** | `test:known-failures:parser` + `test:known-failures` | The gate's strict `ISSUE_` parser is sound; every `known_failure` marker names a real, OPEN issue and is referenced by a test (DB-free, needs `gh`) | no | no | -| **doc-anchors** | `test:doc-anchors` | Every intra-document markdown anchor link resolves to a real heading (DB-free; not relevance-gated — its inputs are the docs) | no | no | -| **validate** (per PG) | `docs:validate:documented-sql` + `test:clean_install_v3` | DB-backed SQL doc-syntax check; clean-DB `eql_v3` install smoke | yes | no | -| **docs-static** | `docs:validate:source` + `test:docs_v3_grep` + `test:public_identifiers` | SQL doxygen coverage + required tags; user docs teach only v3; tracked public files contain no private issue identifiers or tracker links | no | no | -| **schema** | `test:schema` | v2.2 / v2.3 payload JSON-schema validation | no | no | -| **rust-crates** | `test:crates` + `types:check` | `cargo fmt --check`, clippy + `cargo test` for `eql-domains` / `eql-codegen` / `eql-tests-macros` / `eql-bindings`; verify TS bindings + JSON schemas are fresh | no | no | -| **codegen** | `codegen:parity` | Regenerate encrypted-domain SQL in place + `git diff` drift gate (committed `src/v3/scalars/` matches the generator) | no | no | -| **self-contained-v3** | `test:self_contained_v3`, `test:installer_complete`, `test:symbol_order_v3`, `test:build_ordering_helpers` | `eql_v3` surface has no `eql_v2` dependency; installer contains every ordered file; singleton symbols defined before use (overloads are resolved exactly by the `clean-install` job's `test:clean_install_v3`) | no | no | -| **matrix-coverage** | `test:matrix:inventory` (+`:jsonb_entry`, `:v3-jsonb`) + `test:matrix:catalog-coverage` | Scalar-matrix test-name snapshots are not silently dropped; catalog surface is covered | no | no | -| **splinter** | `test:splinter` | Supabase/Splinter lints over the installed EQL | yes (PG17) | no | -| **ci-required** | — | aggregator: every needed job is `success`/`skipped` | no | no | - ---- - -## Where each test suite runs - -The `eql_v3` property-test suites (see -`tests/sqlx/tests/encrypted_domain/property/README.md`) land in three different -CI jobs: - -| Suite | Job | Trigger coverage | DB | `CS_*` | Notes | -|---|---|---|---|---|---| -| **catalog** (`eql-domains` `proptest_invariants`) | **rust-crates** (`cargo test -p eql-domains`; proptest is a dev-dep) | relevant PR + queue | no | no | pure-Rust catalog invariants; shrinking enabled | -| **fixture** (function-double oracles, extractor identity, `match_smoke`, `edge_cases`) | **test** shards (default features) | relevant PR (PG17×4) + queue (PG14–17×2) | yes | no | oracle over the **committed** real-ciphertext fixtures | -| **e2e** (`e2e_oracle`, `#[cfg(feature = "proptest-e2e")]`) | **e2e** job (`test:sqlx:e2e`) | relevant PR (PG17) + queue (PG17) | yes | yes | oracle over **fresh** ZeroKMS encryption; PG-version-independent, so one PG17 run | -| **float specials** (`float_special`, `#[cfg(feature = "proptest-e2e")]`) | **e2e** job (`test:sqlx:e2e`) | relevant PR (PG17) + queue (PG17) | yes | yes | NaN / ±0.0 / ±Inf encoder tripwires over **fresh** encryption | - -The wider sqlx suite (everything under `tests/sqlx/tests/`) runs in the **test** -shards, which replay the default-feature archive — so any -`#[cfg(feature = …)]`-gated test that isn't in the default feature set does not -run there. `proptest-e2e` is the one such gate, and it has its own **e2e** job -(it can't reuse the credential-free archive: it both compiles with a non-default -feature and needs `CS_*` at run time). - -> **The e2e job filters by test name, so every `proptest-e2e` module must be -> named there explicitly.** `test:sqlx:e2e` runs `cargo test --features -> proptest-e2e e2e_oracle` *and* `… float_special`. A bare unfiltered run would -> duplicate the whole sharded fixture suite; but for a long time the job filtered -> on `e2e_oracle` alone, and `float_special` — gated behind the same feature, so -> never compiled by the default-feature shards either — ran in **no** CI job at -> all from the day it was written. Adding a new `proptest-e2e`-gated module means -> adding a line to `test:sqlx:e2e`. Nothing globs it for you. - ---- - -## Known gaps - -1. **bench + macro-expand are nightly / non-blocking** — a bench regression or a - stale `cargo expand` snapshot surfaces on the daily schedule, not on the PR - that introduced it. Accepted trade-off. - -2. **`docs/**` markdown is not content-validated.** The `docs-static` job checks - the SQL `--!` doxygen comments under `src/**`, not the prose/links in `docs/**` - itself. A markdown-only PR leaves `relevant` false, so `docs-static` is skipped - along with the other heavy jobs — and that loses no coverage, because the job's - inputs (`src/**` `.sql`/`.template`, the `crates/**` codegen build, the - `tasks/docs/**` scripts) are all in the `relevant` filter, so a PR that doesn't - trip `relevant` cannot change its outcome. Linting the markdown the PR actually - changed (prose/links) is a separate, unfilled capability. - -### Recently closed - -- *The e2e (fresh-encryption) suite never ran in CI.* Now covered by the **e2e** - job (`test:sqlx:e2e`), PG17, on relevant PRs + the queue. -- *`docs-static` ran unconditionally on every PR.* It is now relevance-gated like - every other heavy job. Because its inputs are a strict subset of the `relevant` - filter, gating it both makes the workflow consistent (one uniform `if:`) and - drops a redundant codegen build on markdown-only PRs without losing any - coverage. A narrower bespoke `src/**`-only filter was rejected: it would risk a - silent false-green (`ci-required` counts `skipped` as pass) by skipping on a - real input change in `crates/**` or `tasks/docs/**`. - ---- - -## Operator setup (one-time, GitHub UI) - -Settings → Branches → rule for `main`: - -1. **Require merge queue.** -2. **Require status checks to pass** → add **`ci-required` only** (not the - per-shard leaf names). - -Then verify (see `docs/plans/2026-06-09-ci-pr-feedback-sharding-rollout.md`): - -- **Queue a relevant PR** → `merge_group` runs the full gate (8 `Shard …` jobs + - 4 `Validate …` jobs + `build-archive`, `e2e`, `docs-static`, `schema`, - `rust-crates`, `codegen`, `self-contained-v3`, `matrix-coverage`, `splinter`) → - `ci-required` green → PR merges. -- **Open a docs-only PR** → on its `pull_request` run every relevance-gated heavy - job skips (`docs-static` included); `ci-required` reports **Success** (not stuck - *Pending*) because it counts `skipped` as pass, so the PR can be queued. - -## References - -- Merge queue: -- `merge_group` event: -- Required status checks: -- `needs` / `always()` / `join()`: -- Path filtering (`dorny/paths-filter`): -- nextest archive + partitioning: · diff --git a/packages/eql/.github/workflows/_build-docs.yml b/packages/eql/.github/workflows/_build-docs.yml deleted file mode 100644 index 8a30f98ad..000000000 --- a/packages/eql/.github/workflows/_build-docs.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: "Build docs (reusable)" - -# Reusable docs build+attach. Called inline by release.yml (production and -# prerelease paths). The target release already exists: _build-sql.yml creates -# it in the same run before this docs job attaches to it. - -on: - workflow_call: - inputs: - ref: - description: "Git ref/SHA to build docs from. Empty -> default checkout (github.sha)." - required: false - type: string - default: "" - tag: - description: "Full release tag. Empty -> build only, no attach." - required: false - type: string - default: "" - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - -defaults: - run: - shell: bash {0} - -permissions: - contents: write - -jobs: - publish-docs: - runs-on: blacksmith-16vcpu-ubuntu-2204 - name: Build and Publish Documentation - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ inputs.ref }} - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - cache: true - - - name: Install Doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - - name: Generate documentation - env: - TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - mise run docs:generate - mise run docs:generate:markdown -- "${TAG}" - mise run docs:generate:json -- "${TAG}" - - - name: Package documentation - env: - TAG: ${{ inputs.tag }} - run: | - mise run docs:package "${TAG}" - - - name: Upload documentation artifacts - uses: actions/upload-artifact@v4 - with: - name: eql-docs - path: | - release/eql-docs-*.zip - release/eql-docs-*.tar.gz - - - name: Publish documentation to release - if: ${{ inputs.tag != '' }} - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag }} - files: | - release/eql-docs-*.zip - release/eql-docs-*.tar.gz diff --git a/packages/eql/.github/workflows/_build-sql.yml b/packages/eql/.github/workflows/_build-sql.yml deleted file mode 100644 index bd5977978..000000000 --- a/packages/eql/.github/workflows/_build-sql.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: "Build SQL (reusable)" - -# Reusable SQL build+attach. Called inline by release.yml (production and -# prerelease paths) so SQL releases have one build path. - -on: - workflow_call: - inputs: - ref: - description: "Git ref/SHA to build from. Empty -> default checkout (github.sha)." - required: false - type: string - default: "" - tag: - description: "Full release tag (e.g. eql-3.0.0-alpha.2). Empty -> DEV build, no attach." - required: false - type: string - default: "" - attach: - description: "Attach the built .sql artefacts to a GitHub Release." - required: false - type: boolean - default: false - target_commitish: - description: "Non-empty -> create a prerelease at this commit; empty -> attach to the existing release named by tag." - required: false - type: string - default: "" - prerelease: - description: "Mark the created release as a prerelease (create path only)." - required: false - type: boolean - default: false - secrets: - MULTITUDES_ACCESS_TOKEN: - required: false - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - -defaults: - run: - shell: bash {0} - -permissions: - contents: write - -jobs: - build: - runs-on: blacksmith-16vcpu-ubuntu-2204 - name: Build EQL - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ inputs.ref }} - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - cache: true - - - name: Build EQL release - # Strip `eql-` so eql_v3.version() reports bare semver. Empty TAG - # intentionally falls through build.sh's ${usage_version:-DEV} default. - env: - TAG: ${{ inputs.tag }} - run: | - mise run build --version "${TAG#eql-}" - - - name: Upload EQL artifacts - uses: actions/upload-artifact@v4 - with: - name: eql-release - path: | - release/cipherstash-encrypt.sql - release/cipherstash-encrypt-uninstall.sql - - - name: Attach artefacts to existing release - if: ${{ inputs.attach && inputs.target_commitish == '' }} - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag }} - files: | - release/cipherstash-encrypt.sql - release/cipherstash-encrypt-uninstall.sql - - - name: Create prerelease at commit - if: ${{ inputs.attach && inputs.target_commitish != '' }} - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag }} - target_commitish: ${{ inputs.target_commitish }} - prerelease: ${{ inputs.prerelease }} - name: ${{ inputs.tag }} - body: "Preview (prerelease) of the standalone eql_v3 surface. See [Unreleased] in CHANGELOG.md." - files: | - release/cipherstash-encrypt.sql - release/cipherstash-encrypt-uninstall.sql - - - name: Notify Multitudes - # Production deploy tracking: fire only when release.yml creates a - # non-prerelease SQL release (attach + create path + not a prerelease). - # release.yml runs on `push`, so the old `event_name == 'release'` gate - # never matched under the unified workflow. - # continue-on-error: a deploy-tracking ping is not part of the release - # itself. If Multitudes is down / rate-limits / the token is empty, the - # curl failing must not fail build-sql and skip downstream build-docs / - # build-image after npm + SQL have already published. - if: ${{ inputs.attach && !inputs.prerelease && inputs.target_commitish != '' }} - continue-on-error: true - run: | - curl --request POST \ - --fail-with-body \ - --url "https://api.developer.multitudes.co/deployments" \ - --header "Content-Type: application/json" \ - --header "Authorization: ${{ secrets.MULTITUDES_ACCESS_TOKEN }}" \ - --data '{"commitSha": "${{ github.sha }}", "environmentName":"production"}' diff --git a/packages/eql/.github/workflows/lint-release.yml b/packages/eql/.github/workflows/lint-release.yml deleted file mode 100644 index b63c21091..000000000 --- a/packages/eql/.github/workflows/lint-release.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: "Lint release tooling" - -# Durable gate for the release machinery. It catches workflow syntax, wrapper -# shell issues, and supply-chain cache regressions before a real release. - -on: - pull_request: - paths: - - .github/workflows/_build-sql.yml - - .github/workflows/_build-docs.yml - - .github/workflows/release-plz.yml - - .github/workflows/release.yml - - .github/workflows/lint-release.yml - - .github/actionlint.yaml - - package.json - - scripts/*.mjs - - tasks/release/*.sh - workflow_dispatch: {} - -permissions: - contents: read - -defaults: - run: - shell: bash - -jobs: - lint: - name: actionlint + shellcheck + unit test - runs-on: blacksmith-16vcpu-ubuntu-2204 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Install actionlint - run: | - set -euo pipefail - bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.7/scripts/download-actionlint.bash) 1.7.7 - echo "$PWD" >> "$GITHUB_PATH" - - - name: actionlint (release workflows) - run: | - set -euo pipefail - actionlint \ - .github/workflows/_build-sql.yml \ - .github/workflows/_build-docs.yml \ - .github/workflows/release-plz.yml \ - .github/workflows/release.yml \ - .github/workflows/lint-release.yml - - - name: shellcheck (wrappers + helpers) - run: | - set -euo pipefail - shellcheck \ - tasks/release/prepare-bindings-assets.sh \ - tasks/release/prepare-bindings-assets.test.sh - - - name: prepare-bindings-assets validation unit test - run: bash tasks/release/prepare-bindings-assets.test.sh - - - uses: pnpm/action-setup@v6.0.8 - name: Install pnpm - with: - run_install: false - cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Install JS dependencies - run: pnpm install --frozen-lockfile - - - name: release workflow supply-chain cache guard - run: | - pnpm run lint:workflow-cache - pnpm run test:scripts diff --git a/packages/eql/.github/workflows/rebuild-docs.yml b/packages/eql/.github/workflows/rebuild-docs.yml deleted file mode 100644 index 3353299d9..000000000 --- a/packages/eql/.github/workflows/rebuild-docs.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Rebuild Docs - -on: - push: - tags: - - 'eql-*' - # Exclude the eql-bindings crate tags (eql-bindings-v*) cut by release-plz - # — a Rust-crate release must not fire the docs-site rebuild webhook. - - '!eql-bindings-*' - -jobs: - trigger-docs-rebuild: - name: Trigger Docs Rebuild - runs-on: blacksmith-16vcpu-ubuntu-2204 - steps: - - name: Send webhook - env: - WEBHOOK_URL: ${{ secrets.DOCS_WEBHOOK_URL }} - run: | - curl -X POST "$WEBHOOK_URL" diff --git a/packages/eql/.github/workflows/release-plz.yml b/packages/eql/.github/workflows/release-plz.yml deleted file mode 100644 index a24248619..000000000 --- a/packages/eql/.github/workflows/release-plz.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: "Release eql-bindings (crates.io)" - -# Publishes the `eql-bindings` crate to crates.io via release-plz (crates.io -# Trusted Publishing via OIDC — no long-lived token; GPG-signed release -# commits/tags; blacksmith runner + mise toolchain). -# -# Publish-only: crate versioning is owned by CHANGESETS, not release-plz. The -# changesets "Version Packages" PR bumps crates/eql-bindings/Cargo.toml to the -# lockstep version V (scripts/sync-lockstep-versions.mjs) alongside the npm -# package. On push to main, `release` publishes the committed Cargo.toml version -# V to crates.io (no-op when V is already published) and tags eql-bindings-vV. -# There is deliberately NO release-plz `release-pr` job — changesets opens the -# version PR, so a release-plz PR would fight it. -# -# One-time crates.io setup (Trusted Publishing): on the eql-bindings crate's -# Settings -> Trusted Publishing, add a GitHub publisher with -# Repository: cipherstash/encrypt-query-language Workflow: release-plz.yml -# For a brand-new crate name, register the publisher before the first publish. -# -# Secrets: GPG_PRIVATE_KEY (release-commit/tag signing key). crates.io auth is -# OIDC (id-token: write), so NO CARGO_REGISTRY_TOKEN. The GitHub PR/tag ops use -# the default GITHUB_TOKEN; enable "Allow GitHub Actions to create and approve -# pull requests" in repo settings so the release PR can be opened. - -permissions: - pull-requests: write - contents: write - id-token: write # crates.io Trusted Publishing (OIDC) - -on: - push: - branches: - - main - workflow_dispatch: {} - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - -# No `defaults.run.shell` override: GitHub's default `bash` shell runs with -# `-eo pipefail` — the right default for a publish workflow. - -# Static group: push-to-main and workflow_dispatch share it so two release-plz -# invocations never race a tag/publish. Never cancel — a cancelled release could -# leave a half-published state. -concurrency: - group: release-plz - cancel-in-progress: false - -jobs: - release: - name: "Release" - runs-on: blacksmith-16vcpu-ubuntu-2204 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Sign release-plz's commits and tags with the shared release key. - - name: Import GPG key - uses: crazy-max/ghaction-import-gpg@v7 - with: - gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - git_user_signingkey: true - git_commit_gpgsign: true - git_tag_gpgsign: true - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - # Publish workflows must not restore caches (lint-no-workflow-caching - # covers mise-action): a poisoned toolchain cache would run inside the - # job that holds crates.io OIDC publishing power. - cache: false - - # The crate bundles its SQL via include_str! on COMMITTED files, and - # release-plz publishes the committed tree verbatim. Refuse to publish a - # crate whose bundled SQL wasn't prepared for this exact version — e.g. a - # hand-pinned Cargo.toml without running `pnpm run version` would - # otherwise ship the DEV placeholder as its "exact SQL". Enforced only - # when this run would actually publish (the committed version is not on - # crates.io yet): routine no-op runs on main between releases carry the - # DEV placeholder legitimately. A crates.io API failure fails towards - # ENFORCING the guard, never towards skipping it. - - name: Verify bundled SQL matches the crate version - run: | - set -euo pipefail - cargo_version="$(grep -m1 '^version = ' crates/eql-bindings/Cargo.toml | cut -d'"' -f2)" - # crates.io rejects curl's default User-Agent with a 403. Without an - # explicit UA the request always fails, `|| true` swallows it, and the - # no-op short-circuit below is unreachable. - published="$(curl -fsSL --retry 3 -H "User-Agent: cipherstash-eql-release (https://github.com/cipherstash/encrypt-query-language)" "https://crates.io/api/v1/crates/eql-bindings/versions" | jq -r '.versions[].num' | grep -Fx "$cargo_version" || true)" - if [[ -n "$published" ]]; then - echo "eql-bindings@${cargo_version} is already on crates.io; this run is a publish no-op — skipping the bundled-SQL guard" - exit 0 - fi - manifest_version="$(jq -r '.eqlVersion' crates/eql-bindings/sql/release-manifest.json)" - if [[ "$manifest_version" != "$cargo_version" ]]; then - echo "::error::crates/eql-bindings/sql/release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml version ('$cargo_version') — the bundled SQL was not prepared for this release. Run 'pnpm run version' (or 'mise run release:prepare_bindings_assets --version $cargo_version') and commit the result." >&2 - exit 1 - fi - if ! grep -q "eql_v3" crates/eql-bindings/sql/cipherstash-encrypt.sql; then - echo "::error::crates/eql-bindings/sql/cipherstash-encrypt.sql looks like the DEV placeholder — refusing to publish it as release SQL." >&2 - exit 1 - fi - - - name: Run release-plz release - uses: release-plz/action@v0.5 - with: - command: release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/packages/eql/.github/workflows/release-postgres-eql-image.yml b/packages/eql/.github/workflows/release-postgres-eql-image.yml deleted file mode 100644 index a311fd703..000000000 --- a/packages/eql/.github/workflows/release-postgres-eql-image.yml +++ /dev/null @@ -1,169 +0,0 @@ -name: "Release Postgres + EQL image" - -on: - # Driven by release.yml on production finals (it dispatches this workflow with - # update_floating_tags=true), plus manual runs. Not `on: release`: a - # GITHUB_TOKEN-created release does not trigger `on: release` fan-out, whereas - # a GITHUB_TOKEN workflow_dispatch does — so release.yml dispatches instead. - workflow_dispatch: - inputs: - eql_version: - description: "EQL version to embed (e.g. 2.1.8). Used as the image tag and SQL build version." - required: true - type: string - update_floating_tags: - description: "Also update :latest, :, and : floating tags. Off by default for test runs." - required: false - type: boolean - default: false - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - IMAGE: ghcr.io/cipherstash/postgres-eql - -defaults: - run: - shell: bash {0} - -permissions: - contents: read - packages: write - -jobs: - build-sql: - runs-on: blacksmith-16vcpu-ubuntu-2204 - name: Build EQL SQL - timeout-minutes: 5 - outputs: - # Used for image tags, e.g. "2.1.8" -> ghcr.io/.../postgres-eql:17-2.1.8 - eql_version: ${{ steps.ver.outputs.eql_version }} - # Passed to `mise run build --version`. Bare semver (no `eql-` prefix), - # matching the SQL release build (_build-sql.yml), so eql_v3.version() in - # the image is byte-identical to the SQL release. - build_version: ${{ steps.ver.outputs.build_version }} - update_floating_tags: ${{ steps.ver.outputs.update_floating_tags }} - - steps: - - uses: actions/checkout@v4 - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - cache: true - - - name: Compute EQL version and tag policy - id: ver - # eql_version is passed in as bare semver (release.yml sends - # needs.release.outputs.version, e.g. "3.0.0"), used both as the image - # tag and, verbatim, as the `mise run build --version` value so - # eql_v3.version() in the image is byte-identical to the SQL release. - env: - INPUT_VERSION: ${{ inputs.eql_version }} - INPUT_FLOATING: ${{ inputs.update_floating_tags }} - run: | - { - echo "eql_version=${INPUT_VERSION}" - echo "build_version=${INPUT_VERSION}" - echo "update_floating_tags=${INPUT_FLOATING}" - } >> "$GITHUB_OUTPUT" - - - name: Build EQL release SQL - run: mise run build --version ${{ steps.ver.outputs.build_version }} - - - name: Upload EQL SQL artifact - uses: actions/upload-artifact@v4 - with: - name: eql-sql - path: release/cipherstash-encrypt.sql - - build-images: - runs-on: blacksmith-16vcpu-ubuntu-2204 - needs: build-sql - name: Build postgres-eql image (PG ${{ matrix.pg_version }}) - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - pg_version: ["14", "15", "16", "17"] - env: - EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} - UPDATE_FLOATING: ${{ needs.build-sql.outputs.update_floating_tags }} - - steps: - - uses: actions/checkout@v4 - - - name: Download EQL SQL artifact - uses: actions/download-artifact@v4 - with: - name: eql-sql - path: docker/ - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Compute image tags - id: tags - # Always push the immutable :- tag. - # Only push the floating : tag on release, or on dispatch with opt-in. - run: | - tags="${IMAGE}:${{ matrix.pg_version }}-${EQL_VERSION}" - if [[ "${UPDATE_FLOATING}" == "true" ]]; then - tags+=$'\n'"${IMAGE}:${{ matrix.pg_version }}" - fi - { - echo 'tags<> "$GITHUB_OUTPUT" - - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: docker - file: docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - build-args: | - PG_VERSION=${{ matrix.pg_version }} - EQL_VERSION=${{ env.EQL_VERSION }} - tags: ${{ steps.tags.outputs.tags }} - - promote-latest: - runs-on: blacksmith-16vcpu-ubuntu-2204 - needs: [build-sql, build-images] - name: Promote PG17 image to :latest and : - if: ${{ needs.build-sql.outputs.update_floating_tags == 'true' }} - timeout-minutes: 5 - env: - EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} - - steps: - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Re-tag PG17 image as :latest and : - # buildx imagetools preserves multi-arch manifests; `docker tag` would not. - run: | - docker buildx imagetools create \ - -t "${IMAGE}:latest" \ - -t "${IMAGE}:${EQL_VERSION}" \ - "${IMAGE}:17-${EQL_VERSION}" diff --git a/packages/eql/.github/workflows/release.yml b/packages/eql/.github/workflows/release.yml deleted file mode 100644 index 9067cf853..000000000 --- a/packages/eql/.github/workflows/release.yml +++ /dev/null @@ -1,426 +0,0 @@ -name: "Release" - -# Single release entry point for the EQL release line. -# -# - `main` uses Changesets to version and publish the production release. -# - Any non-`main` branch can cut a prerelease from an explicit conventional -# release commit, triggered via workflow_dispatch against that branch (e.g. -# `gh workflow run release.yml --ref eql-3.0.1`). The release commit itself is -# the marker: `chore(release): ...`. This lets a batching branch ship an -# alpha/rc for pre-merge testing without merging to `main`. -# -# Prereleases publish the npm package directly from this workflow and dispatch -# the Rust crate publish through `release-plz.yml` so crates.io Trusted -# Publishing still sees the correct entry-point workflow. The SQL GitHub -# release and docs are built in the same workflow run. -# -# npm Trusted Publishing must be configured for this workflow filename: -# `release.yml`. - -permissions: - id-token: write # npm OIDC trusted publishing - contents: write # changesets commits the Version PR / tags; SQL release attach - pull-requests: write # changesets opens the Version PR - -on: - push: - branches: - - main - workflow_dispatch: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -# No `defaults.run.shell` override: GitHub's default `bash` shell runs with -# `-eo pipefail`, which is exactly what a publish workflow wants. (An explicit -# `bash {0}` would silently DISABLE errexit/pipefail for any step that forgets -# its own `set -euo pipefail`.) - -jobs: - classify: - name: Classify release intent - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - mode: ${{ steps.classify.outputs.mode }} - version: ${{ steps.classify.outputs.version }} - prerelease: ${{ steps.classify.outputs.prerelease }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - id: classify - run: | - set -euo pipefail - branch="${GITHUB_REF_NAME}" - subject="$(git log -1 --format=%s)" - mode="skip" - version="" - prerelease="false" - - if [[ "$branch" == "main" ]]; then - mode="production" - else - # Any non-main branch cuts a prerelease from an explicit - # conventional release commit (dispatched via workflow_dispatch). - # Safety comes from the three guards below — the marker commit, the - # prerelease-shaped version, and the tag-idempotency short-circuit — - # not from the branch name. Exactly `chore(release):`; a bare - # `release:` prefix must NOT re-trigger the publish pipeline for the - # currently pinned identity. - case "$subject" in - chore\(release\):*) - mode="prerelease" - ;; - esac - fi - - if [[ "$mode" == "prerelease" ]]; then - version="$(node -p "require('./packages/eql/package.json').version")" - if [[ "$version" != *-* ]]; then - echo "::error::prerelease release commits must already pin a prerelease version in packages/eql/package.json" >&2 - exit 1 - fi - # Idempotency short-circuit: if this identity's npm tag already - # exists, the release was already cut — a re-push of the marker (or - # a rerun) must not re-publish. Individual steps are also - # idempotent; this just skips the whole pipeline up front. - if git ls-remote --exit-code --tags origin "refs/tags/eql-typescript-v${version}" >/dev/null 2>&1; then - echo "eql-typescript-v${version} already exists; prerelease ${version} was already published — skipping" - mode="skip" - version="" - else - prerelease="true" - fi - fi - - { - echo "mode=$mode" - echo "version=$version" - echo "prerelease=$prerelease" - } >> "$GITHUB_OUTPUT" - - release: - name: Version or publish (changesets) - needs: classify - if: ${{ needs.classify.outputs.mode == 'production' }} - # GitHub-hosted (not Blacksmith): npm provenance from OIDC trusted - # publishing is only accepted from github-hosted runners (self-hosted -> E422). - runs-on: ubuntu-latest - timeout-minutes: 20 - outputs: - published: ${{ steps.changesets.outputs.published }} - version: ${{ steps.ver.outputs.version }} - prerelease: ${{ steps.ver.outputs.prerelease }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: pnpm/action-setup@v6.0.8 - name: Install pnpm - with: - run_install: false - cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - # Publish workflows must not restore caches (lint-no-workflow-caching - # covers mise-action too): a poisoned toolchain cache would run inside - # the job that holds npm OIDC publishing power. - cache: false - - # AFTER mise-action, deliberately: mise's node (from [tools]) is first on - # PATH and its bundled npm 10.x has no OIDC trusted publishing — upgrading - # before mise-action upgrades an npm that never runs (the exact cause of - # the alpha.3 ENEEDAUTH failures). - - name: Upgrade npm for OIDC trusted publishing - run: npm install -g npm@11.5.1 - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - # `version:` runs `changeset version && sync-lockstep-versions.mjs` (bumps - # package.json + Cargo.toml + builds/bundles SQL to V) for the Version PR; - # `publish:` runs `pnpm run build && changeset publish` to publish npm. - - name: Version or publish - id: changesets - uses: changesets/action@v1.8.0 - with: - version: pnpm run version - publish: pnpm run release - commitMode: "github-api" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Resolve published version - id: ver - if: steps.changesets.outputs.published == 'true' - env: - PUBLISHED: ${{ steps.changesets.outputs.publishedPackages }} - run: | - set -euo pipefail - version="$(node -e "const p=JSON.parse(process.env.PUBLISHED);const e=p.find(x=>x.name==='@cipherstash/eql');if(!e){process.exit(1)}console.log(e.version)")" - echo "version=${version}" >> "$GITHUB_OUTPUT" - if [[ "$version" == *-* ]]; then - echo "prerelease=true" >> "$GITHUB_OUTPUT" - else - echo "prerelease=false" >> "$GITHUB_OUTPUT" - fi - - build-sql: - name: Build + attach SQL release - needs: release - if: ${{ needs.release.outputs.published == 'true' }} - permissions: - contents: write - secrets: - MULTITUDES_ACCESS_TOKEN: ${{ secrets.MULTITUDES_ACCESS_TOKEN }} - uses: ./.github/workflows/_build-sql.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.release.outputs.version }} - attach: true - target_commitish: ${{ github.sha }} - prerelease: ${{ needs.release.outputs.prerelease == 'true' }} - - build-docs: - name: Build + attach docs - needs: [release, build-sql] - if: ${{ needs.release.outputs.published == 'true' }} - permissions: - contents: write - uses: ./.github/workflows/_build-docs.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.release.outputs.version }} - - build-image: - name: Dispatch Postgres + EQL image build - needs: [release, build-sql, build-docs] - # Production finals only: the floating :latest / : image tags must - # not move for a prerelease. Alpha images can still be built on demand via - # release-postgres-eql-image.yml's workflow_dispatch. Dispatched (not an - # `on: release` fan-out) because a GITHUB_TOKEN-created release does not - # trigger `on: release`, whereas a GITHUB_TOKEN workflow_dispatch does. - if: ${{ needs.release.outputs.published == 'true' && needs.release.outputs.prerelease == 'false' }} - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - actions: write - steps: - - name: Dispatch release-postgres-eql-image.yml - # Dispatch against the eql- tag, not the branch: build-sql - # (a `needs`) already created that tag at the release commit via - # _build-sql.yml's target_commitish, and the image workflow file exists - # at that commit. Pinning to the tag builds the image from the exact - # released source even if the branch has advanced since publish. - env: - GH_TOKEN: ${{ github.token }} - VERSION: ${{ needs.release.outputs.version }} - run: | - set -euo pipefail - # This job deliberately has no checkout. Pass --repo explicitly so - # gh does not try to infer the repository from a nonexistent .git - # directory. - gh workflow run release-postgres-eql-image.yml \ - --repo "${GITHUB_REPOSITORY}" \ - --ref "eql-${VERSION}" \ - -f eql_version="$VERSION" \ - -f update_floating_tags=true - - prerelease-build-sql: - name: Build + attach prerelease SQL - needs: classify - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - permissions: - contents: write - secrets: - MULTITUDES_ACCESS_TOKEN: ${{ secrets.MULTITUDES_ACCESS_TOKEN }} - uses: ./.github/workflows/_build-sql.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.classify.outputs.version }} - attach: true - target_commitish: ${{ github.sha }} - prerelease: true - - prerelease-build-docs: - name: Build + attach prerelease docs - needs: [classify, prerelease-build-sql] - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - permissions: - contents: write - uses: ./.github/workflows/_build-docs.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.classify.outputs.version }} - - prerelease-publish-npm: - name: Publish prerelease npm package - needs: [classify, prerelease-build-sql, prerelease-build-docs] - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: pnpm/action-setup@v6.0.8 - name: Install pnpm - with: - run_install: false - cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - # Publish workflows must not restore caches (lint-no-workflow-caching - # covers mise-action too): a poisoned toolchain cache would run inside - # the job that holds npm OIDC publishing power. - cache: false - - # AFTER mise-action, deliberately: mise's node (from [tools]) is first on - # PATH and its bundled npm 10.x has no OIDC trusted publishing — upgrading - # before mise-action upgrades an npm that never runs (the exact cause of - # the alpha.3 ENEEDAUTH failures). - - name: Upgrade npm for OIDC trusted publishing - run: npm install -g npm@11.5.1 - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Verify prerelease marker and version - env: - VERSION: ${{ needs.classify.outputs.version }} - run: | - set -euo pipefail - actual="$(node -p "require('./packages/eql/package.json').version")" - test "$actual" = "$VERSION" || { - echo "package version ${actual} does not match prerelease identity ${VERSION}" >&2 - exit 1 - } - - - name: Prepare exact SQL assets - env: - VERSION: ${{ needs.classify.outputs.version }} - run: mise run release:prepare_bindings_assets --version "$VERSION" - - - name: Build package - run: pnpm --filter @cipherstash/eql build - - - name: Publish package - working-directory: packages/eql - env: - VERSION: ${{ needs.classify.outputs.version }} - run: | - set -euo pipefail - if [ -n "$(npm view "@cipherstash/eql@${VERSION}" version 2>/dev/null)" ]; then - echo "@cipherstash/eql@${VERSION} is already published; skipping publish" - else - node scripts/npm-publish.mjs - fi - - - name: Tag TypeScript release - env: - VERSION: ${{ needs.classify.outputs.version }} - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - # Create the tag via the REST API with GH_TOKEN — the checkout has - # persist-credentials: false (deliberate hardening) and a git-level - # extraheader hack proved unreliable ("could not read Username"). - tag="eql-typescript-v${VERSION}" - if gh api "repos/${REPO}/git/ref/tags/${tag}" >/dev/null 2>&1; then - echo "tag ${tag} already exists on the remote; skipping" - else - gh api -X POST "repos/${REPO}/git/refs" \ - -f ref="refs/tags/${tag}" -f sha="$GITHUB_SHA" >/dev/null - echo "created ${tag} at ${GITHUB_SHA}" - fi - - prerelease-publish-rust: - name: Dispatch prerelease Rust publish - needs: [classify, prerelease-build-sql, prerelease-build-docs] - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - actions: write - contents: write # create/update the release/eql- branch ref - steps: - - name: Dispatch release-plz.yml at the release commit - # release-plz refuses detached HEADs ("cannot determine current - # branch"), so a tag dispatch does not work. To still publish from the - # EXACT commit the SQL + npm artifacts shipped from — even if eql_v3 - # has advanced since the marker push — pin a release/eql- - # branch at this run's commit and dispatch against that. The branch is - # a stable pointer per identity: force-updated on retries of the same - # identity, never reused across identities. - env: - GH_TOKEN: ${{ github.token }} - VERSION: ${{ needs.classify.outputs.version }} - REPO: ${{ github.repository }} - SHA: ${{ github.sha }} - run: | - set -euo pipefail - branch="release/eql-${VERSION}" - if ! gh api -X POST "repos/${REPO}/git/refs" \ - -f ref="refs/heads/${branch}" -f sha="$SHA" >/dev/null 2>&1; then - gh api -X PATCH "repos/${REPO}/git/refs/heads/${branch}" \ - -f sha="$SHA" -F force=true >/dev/null - fi - # --repo is required: this job has no checkout, and without it gh - # tries to resolve the workflow through a local git repo and dies - # with "not a git repository". - gh workflow run release-plz.yml --repo "$REPO" --ref "$branch" - - summary: - name: Summary - runs-on: ubuntu-latest - needs: [classify, release, build-sql, build-docs, build-image, prerelease-build-sql, prerelease-build-docs, prerelease-publish-npm, prerelease-publish-rust] - if: always() - steps: - - name: Emit run summary - env: - MODE: ${{ needs.classify.outputs.mode }} - VERSION: ${{ needs.classify.outputs.version }} - CLASSIFY_RESULT: ${{ needs.classify.result }} - RELEASE_RESULT: ${{ needs.release.result }} - BUILD_SQL_RESULT: ${{ needs.build-sql.result }} - BUILD_DOCS_RESULT: ${{ needs.build-docs.result }} - BUILD_IMAGE_RESULT: ${{ needs.build-image.result }} - PRE_BUILD_SQL_RESULT: ${{ needs.prerelease-build-sql.result }} - PRE_BUILD_DOCS_RESULT: ${{ needs.prerelease-build-docs.result }} - PRE_PUBLISH_NPM_RESULT: ${{ needs.prerelease-publish-npm.result }} - PRE_PUBLISH_RUST_RESULT: ${{ needs.prerelease-publish-rust.result }} - SERVER_URL: ${{ github.server_url }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} - run: | - set -euo pipefail - { - echo "## Release result" - echo "" - echo "- mode: \`${MODE}\`" - echo "- version: \`${VERSION}\`" - echo "- classify: ${CLASSIFY_RESULT} | release: ${RELEASE_RESULT} | build-sql: ${BUILD_SQL_RESULT} | build-docs: ${BUILD_DOCS_RESULT} | build-image: ${BUILD_IMAGE_RESULT} | prerelease-build-sql: ${PRE_BUILD_SQL_RESULT} | prerelease-build-docs: ${PRE_BUILD_DOCS_RESULT} | prerelease-publish-npm: ${PRE_PUBLISH_NPM_RESULT} | prerelease-publish-rust: ${PRE_PUBLISH_RUST_RESULT}" - echo "" - echo "Run: ${SERVER_URL}/${REPOSITORY}/actions/runs/${RUN_ID}" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/packages/eql/DEVELOPMENT.md b/packages/eql/DEVELOPMENT.md index 910fea132..60a65d7e4 100644 --- a/packages/eql/DEVELOPMENT.md +++ b/packages/eql/DEVELOPMENT.md @@ -421,27 +421,12 @@ Changesets assembles `CHANGELOG.md` and computes the version from these files, s do not hand-edit `CHANGELOG.md`. Behaviour callers should be aware of also needs a numbered upgrade note under `docs/upgrading/`. -#### Public documentation updates +#### Public documentation artifacts -When a tag with the `eql-` prefix is pushed (for example, `eql-1.2.3`), the -workflow at `.github/workflows/rebuild-docs.yml` runs and sends a webhook to the -Vercel-hosted public docs site to trigger a rebuild. - -What happens end-to-end: - -- Release EQL builds EQL artifacts and generates API docs (HTML, XML, Markdown). - The Markdown frontmatter includes the release version. -- Rebuild Docs posts to the `DOCS_WEBHOOK_URL` secret, which Vercel uses to kick - off a fresh build of the public docs. -- The public docs site pulls the latest generated reference - (`docs/api/markdown/API.md`) and publishes it under the corresponding version. - -Manual triggers and troubleshooting: - -- You can re-run the "Rebuild Docs" workflow from the Actions tab if a build - fails downstream. -- Ensure the repository secret `DOCS_WEBHOOK_URL` is set and valid; the workflow - simply POSTs to that URL. +EQL releases generate and attach versioned API documentation artifacts (HTML, +XML, and Markdown) alongside the SQL release. The former Vercel-hosted docs site +and its `DOCS_WEBHOOK_URL` rebuild hook have been retired; publishing an EQL +release does not trigger an external documentation deployment. ### dbdev diff --git a/packages/eql/docker/Dockerfile b/packages/eql/docker/Dockerfile index 78abc7633..f9c638a9f 100644 --- a/packages/eql/docker/Dockerfile +++ b/packages/eql/docker/Dockerfile @@ -6,7 +6,7 @@ FROM postgres:${PG_VERSION} ARG PG_VERSION=17 ARG EQL_VERSION=DEV -LABEL org.opencontainers.image.source="https://github.com/cipherstash/encrypt-query-language" +LABEL org.opencontainers.image.source="https://github.com/cipherstash/stack" LABEL org.opencontainers.image.description="PostgreSQL ${PG_VERSION} with CipherStash EQL ${EQL_VERSION} pre-installed" LABEL org.opencontainers.image.licenses="MPL-2.0" LABEL com.cipherstash.eql.version="${EQL_VERSION}" diff --git a/packages/eql/docs/development/releasing.md b/packages/eql/docs/development/releasing.md index 643344d3e..75362279e 100644 --- a/packages/eql/docs/development/releasing.md +++ b/packages/eql/docs/development/releasing.md @@ -169,11 +169,10 @@ Three independent git tag families, all keyed to the same identity `V`: - **`eql-bindings-vV`** — the **`eql-bindings` Rust crate** on crates.io. - **`eql-typescript-vV`** — the **`@cipherstash/eql` npm package**. -The npm dist-tag is `latest` for finals — and, **until 3.0.0 final ships**, -also for prereleases (the alphas are the only release line, so `latest` -tracks the newest alpha; see `PRE_GA_LATEST` in -`packages/eql/scripts/npm-publish.mjs`). After GA, prereleases return to -their channel dist-tag (`alpha` / `beta` / `rc`). Each language package bundles the **exact** self-contained +The npm dist-tag is `latest` for finals and the channel tag (`alpha` / `beta` / +`rc`) for prereleases. Before 3.0.0 GA prereleases also took `latest`, because +the alphas were the only release line; GA has shipped, so `PRE_GA_LATEST` in +`scripts/npm-publish.mjs` is now `false`. Each language package bundles the **exact** self-contained SQL it was generated against (`eql_bindings::sql`; npm `./sql` subpath), so a consumer pins wire types and the matching DDL together. diff --git a/packages/eql/packages/eql/scripts/npm-publish.mjs b/packages/eql/packages/eql/scripts/npm-publish.mjs index ad698e10c..72a12766a 100644 --- a/packages/eql/packages/eql/scripts/npm-publish.mjs +++ b/packages/eql/packages/eql/scripts/npm-publish.mjs @@ -8,13 +8,12 @@ const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const pkg = JSON.parse( readFileSync(resolve(packageRoot, 'package.json'), 'utf8'), ) -// Dist-tag policy (2026-07-08): until 3.0.0 final ships, `latest` tracks the -// newest release INCLUDING prereleases — the 3.0.0 alphas are the only release -// line, so a bare `npm install @cipherstash/eql` should resolve to the newest -// alpha rather than a stale one. Once 3.0.0 GA is published, flip -// PRE_GA_LATEST to false so prereleases go back to their channel dist-tag -// (alpha/beta/rc) and `latest` stays on finals. -const PRE_GA_LATEST = true +// Dist-tag policy. Before 3.0.0 GA this was `true` so `latest` tracked the +// alphas. GA shipped (npm `latest` is 3.0.5), so prereleases now go to their +// channel tag and `latest` stays on finals — otherwise the next alpha would +// move `latest` off the GA release and `npm install @cipherstash/eql` would +// resolve to it. +const PRE_GA_LATEST = false const prerelease = pkg.version.match(/-(alpha|beta|rc)\./) const tag = prerelease && !PRE_GA_LATEST ? prerelease[1] : 'latest' diff --git a/packages/eql/packages/eql/scripts/verify-release-assets.mjs b/packages/eql/packages/eql/scripts/verify-release-assets.mjs index cc49d8d40..44b28150d 100644 --- a/packages/eql/packages/eql/scripts/verify-release-assets.mjs +++ b/packages/eql/packages/eql/scripts/verify-release-assets.mjs @@ -13,10 +13,16 @@ const pkg = JSON.parse( readFileSync(resolve(packageRoot, 'package.json'), 'utf8'), ) +// mise finds config from the cwd and its parents, and there is no mise config +// at the repository root — the EQL subtree carries its own. A bare `mise run` +// from where these messages are read fails with a TRUST error that reads like +// a broken toolchain, so the `cd` is part of the instruction, not decoration. +const REPAIR = `(cd packages/eql && mise run release:prepare_bindings_assets --version ${pkg.version})` + const manifestPath = resolve(packageRoot, 'sql/release-manifest.json') if (!existsSync(manifestPath)) { console.error( - `refusing to publish: ${manifestPath} is missing — run 'mise run release:prepare_bindings_assets --version ${pkg.version}' first`, + `refusing to publish: ${manifestPath} is missing — run '${REPAIR}' first`, ) process.exit(1) } @@ -24,7 +30,7 @@ if (!existsSync(manifestPath)) { const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) if (manifest.eqlVersion !== pkg.version) { console.error( - `refusing to publish: sql/release-manifest.json eqlVersion ('${manifest.eqlVersion}') does not match package.json version ('${pkg.version}') — the bundled SQL was not prepared for this release. Run 'mise run release:prepare_bindings_assets --version ${pkg.version}' and rebuild.`, + `refusing to publish: sql/release-manifest.json eqlVersion ('${manifest.eqlVersion}') does not match package.json version ('${pkg.version}') — the bundled SQL was not prepared for this release. Run '${REPAIR}' and rebuild.`, ) process.exit(1) } @@ -33,9 +39,21 @@ const installSql = readFileSync( resolve(packageRoot, 'sql/cipherstash-encrypt.sql'), 'utf8', ) -if (!installSql.includes('eql_v3')) { +// THE STAMP, not the schema name. This read `installSql.includes('eql_v3')` +// until it was found unable to fire: `eql_v3` appears ~23,000 times in every +// bundle, and a DEV build is no exception — `tasks/build.sh` substitutes only +// `$RELEASE_VERSION`, which lands in this one COMMENT. So the check passed on +// any input at all, including the placeholder it names. +// +// The manifest comparison above cannot cover this either. Both files are +// regenerated together by `release:prepare_bindings_assets`, so a bundle built +// under the wrong version arrives with a manifest that agrees with it. The +// COMMENT is the only record of which build produced these bytes. Same +// reasoning, and the same predicate, as the guard in `release-plz.yml`. +if (!installSql.includes(`COMMENT ON SCHEMA eql_v3 IS '${pkg.version}'`)) { + const stamped = installSql.match(/COMMENT ON SCHEMA eql_v3 IS '[^']*'/)?.[0] console.error( - 'refusing to publish: sql/cipherstash-encrypt.sql looks like the DEV placeholder — the bundled SQL was not prepared for this release.', + `refusing to publish: sql/cipherstash-encrypt.sql was not built for ${pkg.version} — it carries ${stamped ?? 'no version stamp'}. Run '${REPAIR}' and rebuild.`, ) process.exit(1) } diff --git a/packages/eql/release-plz.toml b/packages/eql/release-plz.toml index 5d00d2582..a2fa9426c 100644 --- a/packages/eql/release-plz.toml +++ b/packages/eql/release-plz.toml @@ -1,17 +1,33 @@ # release-plz configuration. Mirrors the canonical cipherstash-suite setup. # -# Scope: this repository's primary artefact is the EQL SQL surface, released -# manually via `eql-` git tags (see `.github/workflows/release-eql.yml` -# and the "Cutting a release" section of CLAUDE.md). release-plz is scoped to -# the one Rust crate we ship to crates.io: `eql-bindings`. Every other workspace -# member is `publish = false` in its Cargo.toml, so release-plz skips it -# automatically (same convention as cipherstash-suite) — no per-package config. +# Scope: release-plz is scoped to the one Rust crate shipped to crates.io, +# `eql-bindings`. Every other member of this workspace is `publish = false` in +# its Cargo.toml, so release-plz skips it automatically — no per-package config. +# +# THE PATHS IN THIS FILE ARE RELATIVE TO THIS DIRECTORY, not to the repository +# root. This subtree was imported into cipherstash/stack at `packages/eql`, so +# `/.github/workflows/release-plz.yml` passes +# `--manifest-path packages/eql/Cargo.toml --config packages/eql/release-plz.toml`. +# `changelog_config` below is only read by the `update` / `release-pr` commands, +# which this repository never runs: versioning is owned by CHANGESETS +# (`scripts/sync-lockstep-versions.mjs` bumps this crate in lockstep with the +# npm package), and the workflow runs `command: release` — publish only. +# +# This header used to point at `.github/workflows/release-eql.yml` and a +# "Cutting a release" section of CLAUDE.md. Neither existed even upstream: the +# workflow was `release.yml`, and this repository has no such CLAUDE.md section. # # The crate is tagged `eql-bindings-v` (release-plz's default workspace -# tag). That tag deliberately does NOT match the EQL SQL-release automation: the -# `eql`-keyed workflows (release-eql, release-postgres-eql-image, rebuild-docs) -# exclude `eql-bindings-*`, so a crate release never triggers an SQL build, -# Docker image, or docs-site rebuild. +# tag). That tag shape is part of the release identity, not a default nobody +# chose: it deliberately does NOT match the `eql-` SQL-release +# automation, so a crate release never fires an SQL build or a Docker image. +# +# In THIS repository that holds for a stronger reason than upstream's, and the +# difference matters if you go looking: upstream also had a deprecated docs-site +# webhook triggered by `eql-*` tags. That workflow was deliberately not ported. +# The SQL and docs artifact builds are jobs inside `release.yml`, and the image +# is dispatched from it by `gh workflow run`, so a tag of any shape reaches none +# of them. [workspace] changelog_config = "cliff.toml" diff --git a/scripts/__tests__/cargo-lock-freshness.test.mjs b/scripts/__tests__/cargo-lock-freshness.test.mjs index a417b0d9c..f37983d3f 100644 --- a/scripts/__tests__/cargo-lock-freshness.test.mjs +++ b/scripts/__tests__/cargo-lock-freshness.test.mjs @@ -43,6 +43,7 @@ import { REPO_ROOT } from './lib/repo-root.mjs' /** Same skip set, and the same reasons, as `lint-no-eql-registry-pins.mjs`. */ const SKIP_DIRS = new Set([ '.git', + '.claude', 'node_modules', 'target', 'dist', diff --git a/scripts/__tests__/cargo-publish-opt-out.test.mjs b/scripts/__tests__/cargo-publish-opt-out.test.mjs index fccf62389..7df256be5 100644 --- a/scripts/__tests__/cargo-publish-opt-out.test.mjs +++ b/scripts/__tests__/cargo-publish-opt-out.test.mjs @@ -4,29 +4,53 @@ import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' /** - * Every crate in the nested Cargo workspace must opt out of crates.io unless it - * is deliberately allowlisted below. + * Every crate in BOTH nested Cargo workspaces must opt out of crates.io unless + * it is deliberately allowlisted below. * - * A crate with no `publish` key is publishable BY DEFAULT, and `protect-ffi` - * carries none: it has never been on crates.io (verified against the registry - * API), it is a cdylib compiled into `index.node` and shipped inside the six - * `@cipherstash/protect-ffi-` npm packages, and it has no - * Rust-consumer identity at all. Nothing today would publish it — but this repo - * is about to grow a crates.io publisher (`eql-bindings`, via release-plz, when - * `cipherstash/encrypt-query-language` is absorbed), and release-plz publishes - * every workspace member that has not opted out. The convention EQL already - * uses, and which this workspace inherits with that import, is exactly one - * publishable crate with every other member explicitly `publish = false`, so - * release-plz needs no per-package configuration. + * A crate with no `publish` key is publishable BY DEFAULT, and release-plz + * publishes every workspace member that has not opted out. The convention is + * exactly one publishable crate per workspace with every other member + * explicitly `publish = false`, so release-plz needs no per-package + * configuration. * - * The list below is the audit surface: adding a name to it means "a future + * ## Why both, and why that took a second pass + * + * This checked `packages/protect-ffi` ONLY, on the reasoning — written in this + * header — that the repo "is about to grow a crates.io publisher + * (`eql-bindings`, via release-plz, when `cipherstash/encrypt-query-language` + * is absorbed)". The absorption happened. The publisher landed as + * `.github/workflows/release-plz.yml`, pointed by `manifest_path` at + * `packages/eql/Cargo.toml` — and the workspace it publishes from was the one + * workspace this file did not read. The check was strictest exactly where + * nothing could publish and absent where something can. + * + * `packages/protect-ffi/crates/protect-ffi` carries no `publish` key and is + * nonetheless correct: it has never been on crates.io (verified against the + * registry API), it is a cdylib compiled into `index.node` and shipped inside + * the six `@cipherstash/protect-ffi-` npm packages, and nothing + * publishes that workspace. It stays un-allowlisted because allowlisting means + * "release-plz will publish this", which is false for it. + * + * Each workspace's list below is the audit surface: adding a name means "a * release-plz run will publish this crate to crates.io". */ -const WORKSPACE = join(REPO_ROOT, 'packages/protect-ffi') - -/** Crates deliberately published to crates.io. Adding a name here is a decision. */ -const PUBLISHABLE = new Set([]) +const WORKSPACES = [ + { + // The crates.io publisher's target. `release-plz.yml` passes + // `manifest_path: packages/eql/Cargo.toml`. + root: 'packages/eql', + publishable: new Set(['crates/eql-bindings']), + // Pinned so an unreadable `members` list fails loudly rather than yielding + // an empty expansion that passes. + expects: 'crates/eql-bindings', + }, + { + root: 'packages/protect-ffi', + publishable: new Set(), + expects: 'crates/protect-ffi', + }, +] /** * The workspace's members, expanded from its own `[workspace] members` list. @@ -38,7 +62,7 @@ const PUBLISHABLE = new Set([]) * expansion found something, so a members list this parser cannot read fails * loudly instead of yielding an empty set that passes. */ -function workspaceMembers() { +function workspaceMembers(WORKSPACE) { const manifest = readFileSync(join(WORKSPACE, 'Cargo.toml'), 'utf8') const block = /^members\s*=\s*\[([^\]]*)\]/m.exec(manifest)?.[1] ?? '' return [...block.matchAll(/"([^"]+)"/g)] @@ -54,20 +78,34 @@ function workspaceMembers() { .sort() } -describe('cargo publish opt-out', () => { - const members = workspaceMembers() +describe.each(WORKSPACES)('cargo publish opt-out ($root)', ({ + root, + publishable, + expects, +}) => { + const workspace = join(REPO_ROOT, root) + const members = workspaceMembers(workspace) // The guard on the scan: a discovery test that enumerates nothing passes // while checking nothing. it('finds the workspace members it means to check', () => { - expect(members).toContain('crates/protect-ffi') + expect(members).toContain(expects) + }) + + // The guard on the allowlist: an entry naming a member that no longer + // exists is an exemption excusing nothing, and it would go on reading as a + // deliberate decision. + it('allowlists only real members', () => { + expect([...publishable].filter((name) => !members.includes(name))).toEqual( + [], + ) }) for (const member of members) { it(`${member} declares publish = false unless allowlisted`, () => { - if (PUBLISHABLE.has(member)) return + if (publishable.has(member)) return const manifest = readFileSync( - join(WORKSPACE, member, 'Cargo.toml'), + join(workspace, member, 'Cargo.toml'), 'utf8', ) expect(manifest).toMatch(/^publish = false$/m) diff --git a/scripts/__tests__/eql-pipeline-armed.test.mjs b/scripts/__tests__/eql-pipeline-armed.test.mjs new file mode 100644 index 000000000..0f320bf6a --- /dev/null +++ b/scripts/__tests__/eql-pipeline-armed.test.mjs @@ -0,0 +1,149 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + EQL_PACKAGE, + eqlPipelineArmed, + frozenReason, +} from '../eql-pipeline-armed.mjs' +import { FROZEN_PUBLISHERS } from '../release-gate.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * The arming switch, and the wiring that makes it mean anything. + * + * Deriving the switch from `FROZEN_PUBLISHERS` buys nothing if a job stops + * reading it, and a dropped `if:` is invisible: the pipeline is inert for other + * reasons too, so an unguarded job behaves exactly like a guarded one right up + * to the cutover, when it publishes. Hence the equality on the wiring below. + */ + +const SCRIPT = join(REPO_ROOT, 'scripts/eql-pipeline-armed.mjs') + +/** + * Every job that must be gated on the switch. + * + * `promote-latest` is absent on purpose: it `needs:` two gated jobs, so it + * cannot run without them, and its own `if:` carries the floating-tag policy. + */ +const GATED_JOBS = [ + '.github/workflows/release-plz.yml / release', + '.github/workflows/release-postgres-eql-image.yml / build-images', + '.github/workflows/release-postgres-eql-image.yml / build-sql', + '.github/workflows/release.yml / eql-docs', + '.github/workflows/release.yml / eql-image', + '.github/workflows/release.yml / eql-sql', + '.github/workflows/release.yml / prerelease-eql-crate', + '.github/workflows/release.yml / prerelease-eql-docs', + '.github/workflows/release.yml / prerelease-eql-npm', + '.github/workflows/release.yml / prerelease-eql-sql', +] + +/** The workflows that run the script, each exposing it as an `armed` output. */ +const SWITCH_WORKFLOWS = [ + '.github/workflows/release.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', +] + +/** The condition text of every job in a workflow, keyed ` / `. */ +function jobConditions(relPath) { + const wf = readWorkflow(relPath) + return Object.entries(wf?.jobs ?? {}).map(([name, job]) => [ + `${relPath} / ${name}`, + String(job?.if ?? '').replace(/\s+/g, ' '), + ]) +} + +function run(env = {}) { + return execFileSync('node', [SCRIPT], { + cwd: REPO_ROOT, + encoding: 'utf8', + env: { ...process.env, ...env }, + }) +} + +describe('the switch answers from FROZEN_PUBLISHERS', () => { + it('is inert while the package is frozen', () => { + expect(eqlPipelineArmed(new Map([[EQL_PACKAGE, 'a reason']]))).toBe(false) + }) + + it('is armed once the entry is gone — the cutover, exercised now', () => { + // Without this the armed branch first executes at the cutover, which is the + // worst moment to discover the switch was inverted. + expect(eqlPipelineArmed(new Map())).toBe(true) + expect(eqlPipelineArmed(new Map([['@cipherstash/other', 'x']]))).toBe(true) + }) + + it('reports the reason from the map, not from a sentence of its own', () => { + expect(frozenReason(new Map([[EQL_PACKAGE, 'because']]))).toBe('because') + expect(frozenReason(new Map())).toBeNull() + }) + + it('matches the live map, whichever state that is in', () => { + // Not "is currently false" — pinning the verdict would make the cutover + // fail here for no reason. + expect(eqlPipelineArmed()).toBe(!FROZEN_PUBLISHERS.has(EQL_PACKAGE)) + }) +}) + +describe('the switch is readable by a workflow', () => { + it('writes `armed=` to GITHUB_OUTPUT', () => { + const dir = mkdtempSync(join(tmpdir(), 'eql-armed-')) + const outputFile = join(dir, 'output') + try { + execFileSync('sh', ['-c', `: > "${outputFile}"`]) + run({ GITHUB_OUTPUT: outputFile }) + expect(readFileSync(outputFile, 'utf8')).toBe( + `armed=${eqlPipelineArmed()}\n`, + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('says which state it is in, and why, on stdout', () => { + const stdout = run() + expect(stdout).toContain(EQL_PACKAGE) + expect(stdout).toContain(eqlPipelineArmed() ? 'ARMED' : 'INERT') + if (!eqlPipelineArmed()) expect(stdout).toContain(frozenReason()) + }) + + it('does not write GITHUB_OUTPUT when there is none', () => { + // A local run is how someone checks the answer before a cutover. + const stdout = run({ GITHUB_OUTPUT: '' }) + expect(stdout).toContain(EQL_PACKAGE) + }) +}) + +describe('every EQL publish job reads it', () => { + it('exposes the switch as a job output in each release workflow', () => { + const missing = SWITCH_WORKFLOWS.filter((relPath) => { + const job = readWorkflow(relPath)?.jobs?.['eql-armed'] + const runsIt = (job?.steps ?? []).some((step) => + String(step?.run ?? '').includes('scripts/eql-pipeline-armed.mjs'), + ) + return !(runsIt && job?.outputs?.armed) + }) + expect( + missing, + 'These workflows publish an EQL artefact and no longer compute the arming switch, so nothing downstream can be gated on it.', + ).toEqual([]) + }) + + it('gates exactly the jobs that publish an EQL artefact', () => { + const gated = SWITCH_WORKFLOWS.flatMap(jobConditions) + .filter(([, condition]) => + condition.includes("needs.eql-armed.outputs.armed == 'true'"), + ) + .map(([id]) => id) + + expect( + gated.sort(), + "The set of jobs gated on the EQL arming switch has changed. A job that LOST its guard will publish on the day the FROZEN_PUBLISHERS entry is deleted, whether or not anyone meant it to — and until then it is indistinguishable from a job that still has one. Spell the guard exactly `needs.eql-armed.outputs.armed == 'true'`.", + ).toEqual(GATED_JOBS) + }) +}) diff --git a/scripts/__tests__/eql-sql-asset-freshness.test.mjs b/scripts/__tests__/eql-sql-asset-freshness.test.mjs index 62f188828..2732b4562 100644 --- a/scripts/__tests__/eql-sql-asset-freshness.test.mjs +++ b/scripts/__tests__/eql-sql-asset-freshness.test.mjs @@ -1,5 +1,11 @@ import { createHash } from 'node:crypto' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' @@ -73,6 +79,20 @@ const { checked, skew } = eqlLockstepSkew({ version: VERSION, }) +describe('the release SQL build stamps the requested version', () => { + it('forces mise to rebuild when only --version changed', () => { + const workflow = readFileSync( + join(REPO_ROOT, '.github/workflows/_build-eql-sql.yml'), + 'utf8', + ) + + const tagExpansion = '${' + 'TAG#eql-}' + expect(workflow).toContain( + `mise run --force build --version "${tagExpansion}"`, + ) + }) +}) + describe('the EQL lockstep artefacts all declare one version', () => { it('reads a version to compare against', () => { // The floor. `readEqlVersion` throws rather than returning undefined, but a diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 22c16e62a..ee1ea0311 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -865,10 +865,10 @@ const CI_EXEMPT_CARGO_TASKS = new Map([ 'test:lint', '`cargo fmt --check` scoped to tests/sqlx. `test:crates` runs `cargo fmt --check` at the EQL workspace ROOT, and tests/sqlx is a workspace member, so the CI run covers exactly these files and four crates more — verified against `cargo fmt --check -v`, which lists packages/eql/tests/sqlx/** among its targets. test-eql.yml says the same thing at the `rust-crates` job, which is where the standalone lint step used to be.', ], - [ - 'docs:generate:json', - 'Generates docs/api/json/eql-manifest.json for the release docs bundle. Its only caller is `packages/eql/.github/workflows/_build-docs.yml` — one of the four unported RELEASE workflows in UNPORTED_DEPOSIT — so this is category (d): it becomes reachable the day that file is ported, and this entry then goes stale and must be deleted. It is also not portable as-is, since it falls back to `mise run docs:generate`, which needs a doxygen binary no job installs. Its cargo step (`cargo run -p eql-codegen dump-catalog`) is separately exercised in CI by `test:matrix:catalog-coverage`.', - ], + // `docs:generate:json` was here as category (d), naming the unported + // `_build-docs.yml` as its sole caller. That file now runs as + // `_build-eql-docs.yml`, so the exemption went stale exactly as it predicted + // and the staleness check below is what forced its removal. ]) /** Cargo tasks a given reachability set leaves unaccounted for. */ @@ -1292,6 +1292,10 @@ describe('every cargo check EQL owns is reached by a root workflow', () => { * actually fails. */ const SOLE_CALLER_WORKFLOWS = [ + // docs:generate:json, reachable from nothing at all before the release port. + // Note what "sole caller" costs here and not for the three below: this runs + // only on a RELEASE, so a break surfaces mid-publish. + `${WORKFLOW_DIR}/_build-eql-docs.yml`, `${WORKFLOW_DIR}/bench-eql.yml`, // test:bench `${WORKFLOW_DIR}/macro-expand-eql.yml`, // test:matrix:expand `${WORKFLOW_DIR}/test-eql.yml`, // most of the suite @@ -1350,63 +1354,89 @@ describe('the guard fails when a workflow stops calling a cargo check', () => { }) /** - * What is still allowed to sit in the deposited `.github`, and why. + * The deposited `.github` is GONE, and must stay gone. + * + * It was a shrinking allowlist of eleven files, emptied by the release port. + * The rule does not expire with it: a workflow under a package executes on no + * event, which looks exactly like a check that passes. * - * The end state is no directory at all — the rule `lintWiring.test.ts` applies - * to protect-ffi's deposit, and for the same reason: a workflow file under a - * package reads as live CI and is not. It cannot be deleted in one step here, - * because four of EQL's ten workflows are the RELEASE machinery. Porting those - * to the repository root is what makes them fire, and it is gated on repointing - * npm and crates.io trusted publishing — an irreversible cutover that has not - * happened. Deleting them first would mean porting from git history at the one - * moment nobody wants to be reconstructing a publish pipeline. + * Where each file went — "deleted" and "ported" are different claims and a diff + * cannot tell them apart: * - * So this is a SHRINKING allowlist, not an exemption list. Each entry names a - * file that has not been ported yet; porting one means deleting it here in the - * same commit. The equality below fails in both directions — a file that comes - * back fails, and so does an entry left behind after its file is gone, which is - * what turns the last removal into "delete the directory" rather than a check - * that quietly stops meaning anything. + * _build-sql.yml -> .github/workflows/_build-eql-sql.yml + * _build-docs.yml -> .github/workflows/_build-eql-docs.yml + * release.yml -> merged into .github/workflows/release.yml + * (npm trusted publishing binds to a + * filename, so it could not stay separate) + * release-plz.yml -> .github/workflows/release-plz.yml, name + * unchanged (crates.io binds to it) + * release-postgres-eql-image.yml -> same filename at the root + * rebuild-docs.yml -> DROPPED: it posted to the deprecated + * DOCS_WEBHOOK_URL for the retired docs + * site. Versioned docs artifacts are still + * built by _build-eql-docs.yml + * lint-release.yml -> merged into the root file of that name + * .github/actionlint.yaml -> DROPPED: the root copy already carries the + * `blacksmith-16vcpu-ubuntu-2204` label + * ISSUE_TEMPLATE/docs-feedback.yml -> DROPPED: the root copy is the same file + * with the Code of Conduct link already + * repointed at cipherstash/stack + * .github/release.yml -> DROPPED: GitHub's release-notes + * categoriser. Nothing here generates notes + * that way — changesets writes CHANGELOGs + * and the FFI release passes `--notes` + * explicitly — so it would be configuration + * that does nothing + * workflows/README.md -> DROPPED: it documented a merge-queue model + * this repository does not have. `main` is + * unprotected and there is no queue (see + * test-eql.yml's header, which checked that + * against the live API), so porting it would + * have installed a confidently wrong + * document at the root. Its still-true half + * — the two release tag families — is in + * AGENTS.md and in packages/eql/release-plz.toml's + * header */ -const UNPORTED_DEPOSIT = [ - // Phase 5 — the release cutover. Each publishes something, and each is inert - // until trusted publishing is repointed at cipherstash/stack. - '.github/release.yml', // release-notes categorisation config, not a workflow - '.github/workflows/README.md', // documents the four below - '.github/workflows/_build-docs.yml', // the sole caller of `docs:generate:json` - '.github/workflows/_build-sql.yml', - '.github/workflows/lint-release.yml', // actionlints the four below - '.github/workflows/rebuild-docs.yml', - '.github/workflows/release-plz.yml', - '.github/workflows/release-postgres-eql-image.yml', - '.github/workflows/release.yml', - // Not CI. Ported with the repository settings, not with a workflow. - '.github/ISSUE_TEMPLATE/docs-feedback.yml', - '.github/actionlint.yaml', -].sort() - -describe('the imported workflow directory is on its way out', () => { - it('holds only the files still waiting to be ported', () => { - const deposit = existsSync(join(REPO_ROOT, DEAD_GITHUB_DIR)) - ? execFileSync('git', ['ls-files', '-z', '--', DEAD_GITHUB_DIR], { - cwd: REPO_ROOT, - encoding: 'utf8', - }) - .split('\0') - .filter(Boolean) - .map((path) => path.slice('packages/eql/'.length)) - .sort() - : [] +describe('the imported workflow directory is gone', () => { + it('does not exist', () => { + expect( + existsSync(join(REPO_ROOT, DEAD_GITHUB_DIR)), + `${DEAD_GITHUB_DIR} is back. GitHub reads workflows from the repository root alone, ` + + 'so anything in there runs on no event — which looks exactly like a check that passes. ' + + 'Put the workflow in .github/workflows/ instead.', + ).toBe(false) + }) + + it('has no tracked files, even if the directory is untracked scratch', () => { + // `existsSync` alone would miss the case that actually matters: a file + // committed back into the deposit by a subtree pull, in a working tree + // where the directory was never removed locally. + const tracked = execFileSync( + 'git', + ['ls-files', '-z', '--', DEAD_GITHUB_DIR], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ) + .split('\0') + .filter(Boolean) + + expect( + tracked, + `These files are tracked under ${DEAD_GITHUB_DIR} and execute on nothing.`, + ).toEqual([]) + }) +}) + +describe('the retired docs site stays retired', () => { + it('has no workflow consuming the deprecated webhook secret', () => { + const consumers = workflowFiles().filter((file) => + readRepo(file).includes('DOCS_WEBHOOK_URL'), + ) expect( - deposit, - `${DEAD_GITHUB_DIR} no longer matches the list of files still waiting to be ported.\n` + - 'If you PORTED one, delete it from the deposit and from UNPORTED_DEPOSIT in the same commit — ' + - 'leaving it here means two copies of a workflow, one of which GitHub ignores.\n' + - 'If you ADDED one, it is inert: GitHub reads workflows from the repository root alone.\n' + - 'When this list empties, delete the directory and replace this check with ' + - '`expect(existsSync(...)).toBe(false)`.', - ).toEqual(UNPORTED_DEPOSIT) + consumers, + 'DOCS_WEBHOOK_URL belonged to the retired docs site; release workflows must publish the versioned docs artifact without invoking that webhook.', + ).toEqual([]) }) }) @@ -1456,6 +1486,52 @@ function reachesCargo(name, seen = new Set()) { return result } +/** + * The workflows that MUST NOT restore a cache, read from the gate that says so. + * + * `lint-no-workflow-caching.mjs` forbids a cache restore in a publishing + * workflow; the check below wants a `Swatinem/rust-cache` step on every job + * that compiles Rust. Four EQL release jobs do both, so they pay a cold + * compile. Derived from the linter's own target list rather than copied, so the + * two cannot disagree about which jobs those are. + * + * Lazy: the linter lints on import, so its targets are read by running it, and + * a real caching violation exits non-zero. Computing this at module scope made + * that a collect failure, taking all 40-odd assertions in this file with it. + */ +let noCacheWorkflows +function inNoCacheWorkflow(relPath) { + if (noCacheWorkflows === undefined) { + let stdout + try { + stdout = execFileSync( + 'node', + [join(REPO_ROOT, 'scripts/lint-no-workflow-caching.mjs')], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ) + } catch (err) { + throw new Error( + 'lint-no-workflow-caching.mjs exited non-zero, so its target list could ' + + 'not be read. Fix the caching finding it reports first:\n' + + String(err?.stdout ?? '') + + String(err?.stderr ?? ''), + ) + } + const targets = stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith(`${WORKFLOW_DIR}/`)) + if (targets.length === 0) { + throw new Error( + 'lint-no-workflow-caching.mjs printed no targets, so the publish-path ' + + 'exemption below would silently cover nothing.', + ) + } + noCacheWorkflows = new Set(targets) + } + return noCacheWorkflows.has(relPath) +} + /** * Every job in a root workflow, with the cargo-reaching mise tasks its steps * invoke and whether it restores the Rust cache. @@ -1512,8 +1588,33 @@ describe('every job that compiles Rust restores the shared cache', () => { ).toBeGreaterThan(0) }) + it('exempts the publish paths, and only those', () => { + // Worth its own assertion: if the derivation stops selecting these jobs + // they rejoin the check below, whose natural repair is a rust-cache step + // the caching gate then rejects. + const exempt = RUST_JOBS.filter(({ relPath }) => inNoCacheWorkflow(relPath)) + expect( + exempt.map(({ relPath, jobName }) => `${relPath} (${jobName})`).sort(), + 'No cargo-compiling job sits in a no-cache publish workflow any more. If the EQL release pipeline moved, move this with it; if it was deleted, delete this.', + ).toEqual([ + '.github/workflows/_build-eql-docs.yml (publish-docs)', + '.github/workflows/_build-eql-sql.yml (build)', + '.github/workflows/release-postgres-eql-image.yml (build-sql)', + '.github/workflows/release.yml (prerelease-eql-npm)', + ]) + + // …and none smuggled one in anyway. The caching gate would catch it too; + // this says so where the reader is already looking. + expect( + exempt.filter(({ cached }) => cached).map(({ relPath }) => relPath), + 'A publish-path job carries a `Swatinem/rust-cache` step. That is a GitHub Actions cache restore inside a job holding a publishing credential — see scripts/lint-no-workflow-caching.mjs.', + ).toEqual([]) + }) + it('leaves no cargo-compiling job uncached', () => { - const uncached = RUST_JOBS.filter(({ cached }) => !cached).map( + const uncached = RUST_JOBS.filter( + ({ relPath, cached }) => !cached && !inNoCacheWorkflow(relPath), + ).map( ({ relPath, jobName, compiling }) => `${relPath} (${jobName}): runs ${compiling.join(', ')}`, ) diff --git a/scripts/__tests__/lib/package-readmes.mjs b/scripts/__tests__/lib/package-readmes.mjs new file mode 100644 index 000000000..fdfe25188 --- /dev/null +++ b/scripts/__tests__/lib/package-readmes.mjs @@ -0,0 +1,43 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { workspacePackagePatterns } from '../../release-gate.mjs' +import { REPO_ROOT } from './repo-root.mjs' + +/** + * git pathspecs selecting every workspace package's `README.md`. + * + * ## Why this is derived rather than written down + * + * Both callers used to hardcode `:(glob)packages/*/README.md`. `:(glob)` stops + * `*` at a path separator — which is what makes `lib/*.ts` behave — and this + * repo has TWO package roots nested deeper than one level: + * `packages/protect-ffi/platforms/*` and `packages/eql/packages/*`. So the + * hardcoded spec selected `packages/eql/README.md`, the 15 KB subtree root that + * ships in no tarball, and never `packages/eql/packages/eql/README.md`, the + * 518-byte file listed in that package's `files`. Same for the six per-platform + * FFI packages. Seven published READMEs were unscanned and one unpublished one + * was standing in for them, which is worse than scanning nothing: the guards + * reported green over a set that did not contain what they were guarding. + * + * `pnpm-workspace.yaml` is the only place that knows how deep a package can be, + * and it has to be edited to add one — so deriving from it means a third nested + * root is covered the day it lands, rather than the day someone remembers this + * file. Same reasoning, and the same parser, as `workspaceManifests()`. + * + * Narrowed to `packages/` deliberately: `examples/*` and `e2e` are workspace + * members too, but they are private and their READMEs are not shipped to + * anyone. The callers are guards on SHIPPED text. + */ +export function packageReadmePathspecs() { + const patterns = workspacePackagePatterns( + readFileSync(join(REPO_ROOT, 'pnpm-workspace.yaml'), 'utf8'), + ).filter((pattern) => pattern.startsWith('packages/')) + + if (patterns.length === 0) { + throw new Error( + 'pnpm-workspace.yaml lists no `packages/` patterns — either the layout moved or this derivation broke. Failing rather than returning an empty pathspec list, which `git ls-files` would answer with the whole tree.', + ) + } + + return patterns.map((pattern) => `:(glob)${pattern}/README.md`) +} diff --git a/scripts/__tests__/lint-no-workflow-caching.test.mjs b/scripts/__tests__/lint-no-workflow-caching.test.mjs index e0fb11f34..fbda6351c 100644 --- a/scripts/__tests__/lint-no-workflow-caching.test.mjs +++ b/scripts/__tests__/lint-no-workflow-caching.test.mjs @@ -17,6 +17,14 @@ import { REPO_ROOT } from './lib/repo-root.mjs' const TARGET_WORKFLOWS = [ '.github/workflows/release.yml', '.github/workflows/_build-ffi-artifacts.yml', + // The EQL release line. `scripts/__tests__/eql-suite-ci.test.mjs` reads the + // script's real list too, and uses it to exempt these workflows from the + // rust-cache requirement — the two rules point opposite ways for a job that + // compiles Rust inside a publish path, and that file records which one wins. + '.github/workflows/_build-eql-sql.yml', + '.github/workflows/_build-eql-docs.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', '.github/workflows/tests-supply-chain.yml', ] diff --git a/scripts/__tests__/no-removed-drizzle-surface.test.mjs b/scripts/__tests__/no-removed-drizzle-surface.test.mjs index 1ee0f063c..b271c58f9 100644 --- a/scripts/__tests__/no-removed-drizzle-surface.test.mjs +++ b/scripts/__tests__/no-removed-drizzle-surface.test.mjs @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' +import { packageReadmePathspecs } from './lib/package-readmes.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' /** @@ -28,7 +29,9 @@ const REMOVED = [ // wildmatch crosses `/` and `lib/*.ts` sweeps in `lib/__tests__/*.test.ts`. const SHIPPED_GLOBS = [ ':(glob)skills/*/SKILL.md', - ':(glob)packages/*/README.md', + // Derived, not written down: two package roots sit deeper than one level and + // `:(glob)` does not cross `/`. See `lib/package-readmes.mjs`. + ...packageReadmePathspecs(), 'README.md', 'AGENTS.md', // `stash init` writes these strings into the user's project as real source. @@ -53,6 +56,14 @@ describe('removed stack-drizzle surface is absent from shipped files', () => { expect(files.length).toBeGreaterThan(5) expect(files).toContain('skills/stash-drizzle/SKILL.md') expect(files).toContain('packages/stack/README.md') + // The two roots nested deeper than `packages/*`. Pinned by name because + // they are what the hardcoded `:(glob)packages/*/README.md` silently + // missed: `:(glob)` does not cross `/`, so it selected the EQL subtree + // ROOT's README — which ships in no tarball — instead of the package's. + expect(files).toContain('packages/eql/packages/eql/README.md') + expect(files).toContain( + 'packages/protect-ffi/platforms/linux-x64-gnu/README.md', + ) }) it.each(files)('%s', (file) => { diff --git a/scripts/__tests__/no-removed-eql-version-flag.test.mjs b/scripts/__tests__/no-removed-eql-version-flag.test.mjs index d2aea7e6a..d11016a38 100644 --- a/scripts/__tests__/no-removed-eql-version-flag.test.mjs +++ b/scripts/__tests__/no-removed-eql-version-flag.test.mjs @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' +import { packageReadmePathspecs } from './lib/package-readmes.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' /** Tracked executable examples that describe the current public surface. */ @@ -12,7 +13,9 @@ function publicCommandDocs() { 'ls-files', '-z', ':(glob)skills/*/SKILL.md', - ':(glob)packages/*/README.md', + // Derived, not written down: two package roots sit deeper than one level + // and `:(glob)` does not cross `/`. See `lib/package-readmes.mjs`. + ...packageReadmePathspecs(), 'docs/reference/supabase-sdk.md', ], { @@ -77,6 +80,14 @@ describe('public eql install examples use the current CLI', () => { expect(files).toContain('skills/stash-supabase/SKILL.md') expect(files).toContain('packages/stack/README.md') expect(files).toContain('docs/reference/supabase-sdk.md') + // The two roots nested deeper than `packages/*`. Pinned by name because + // they are what the hardcoded `:(glob)packages/*/README.md` silently + // missed: `:(glob)` does not cross `/`, so it selected the EQL subtree + // ROOT's README — which ships in no tarball — instead of the package's. + expect(files).toContain('packages/eql/packages/eql/README.md') + expect(files).toContain( + 'packages/protect-ffi/platforms/linux-x64-gnu/README.md', + ) }) it.each(files)('%s', (file) => { diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index 3fc7dee8d..4611d7d5c 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -65,6 +65,37 @@ const EXPECTED_DISPATCHABLE = [ '.github/workflows/lint-release.yml', '.github/workflows/osv-scanner.yml', '.github/workflows/tests-rust.yml', + // The EQL release line. `release.yml` gained a dispatch with the port, + // because the EQL PRERELEASE path is cut by dispatching a batching branch + // whose HEAD is a `chore(release):` marker; the other two are how an image or + // a crate publish is re-run by hand after a partial failure. + '.github/workflows/release.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', +] + +/** + * Jobs that legitimately do NOT run on a plain manual dispatch, with the reason. + * + * The check below required EVERY job-level `if:` to be true under one synthetic + * dispatch context. Exact for a workflow with one path; `release.yml` has two, + * mutually exclusive by construction (`classify` reports `production` for main + * and `prerelease` for a marker commit elsewhere), so under any single context + * one half is false. + * + * Relaxing to "at least one job runs" would say nothing about a ten-job + * workflow where nine skip. The per-job requirement stays; the exceptions + * become an equality, failing in both directions. The synthetic context + * dispatches against `refs/heads/main`, so production is what is modelled. + */ +const DISPATCH_SKIPPED_JOBS = [ + // The four prerelease jobs. A dispatch against `main` classifies as + // `production`, so these are correctly skipped; they run when the dispatch + // names a branch whose HEAD commit subject is `chore(release):`. + '.github/workflows/release.yml / prerelease-eql-crate', + '.github/workflows/release.yml / prerelease-eql-docs', + '.github/workflows/release.yml / prerelease-eql-npm', + '.github/workflows/release.yml / prerelease-eql-sql', ] /** @@ -373,6 +404,26 @@ function runsWhen(condition, context) { */ const PERMISSIVE_NEEDS = { changes: { outputs: { relevant: 'true' } }, + // The release workflows' gates, none of which is about how the run was + // triggered. Held open, or "skips because there is nothing to publish" would + // be indistinguishable from "skips on a dispatch". + classify: { outputs: { mode: 'production', version: '3.0.6' } }, + 'eql-armed': { outputs: { armed: 'true' } }, + gate: { result: 'success', outputs: { ffi: 'true' } }, + 'publish-ffi': { result: 'success' }, + release: { + result: 'success', + outputs: { + eql_published: 'true', + eql_version: '3.0.6', + // A final: `eql-image` runs only for one. The prerelease half of the + // file is in DISPATCH_SKIPPED_JOBS instead. + eql_prerelease: 'false', + }, + }, + // release-postgres-eql-image.yml's `promote-latest` reads this from its own + // `build-sql` job, which copies the dispatch input through. + 'build-sql': { outputs: { update_floating_tags: 'true' } }, } /** @@ -512,7 +563,22 @@ describe('a declared workflow_dispatch actually dispatches', () => { ).not.toEqual([]) }) - for (const { id, condition } of DISPATCHABLE_CONDITIONS) { + it('skips exactly the jobs declared unreachable by a plain dispatch', () => { + // Both directions, so the list cannot become a place findings go to be + // forgotten. + const skipped = DISPATCHABLE_CONDITIONS.filter( + ({ condition }) => !runsWhen(condition, CONTEXTS.workflow_dispatch), + ).map((entry) => entry.id) + + expect( + skipped.sort(), + 'The set of jobs that do not run on a manual dispatch has changed. If a job JOINED it, its `workflow_dispatch:` just became decorative for that job — gate on the case that genuinely cannot run rather than enumerating the events that can. If a job LEFT it, delete its DISPATCH_SKIPPED_JOBS entry in the same commit.', + ).toEqual([...DISPATCH_SKIPPED_JOBS].sort()) + }) + + for (const { id, condition } of DISPATCHABLE_CONDITIONS.filter( + (entry) => !DISPATCH_SKIPPED_JOBS.includes(entry.id), + )) { it(`${id} runs on a manual dispatch`, () => { expect( runsWhen(condition, CONTEXTS.workflow_dispatch), diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index d9557fb18..e4dff0592 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -97,6 +97,20 @@ const EXPECTED_PARITY_WORKFLOWS = [ * reason has to explain why the drift it guards against is not possible. */ const EXPECTED_ASYMMETRIES = new Map([ + [ + '.github/workflows/release-plz.yml', + // There is no `pull_request:` trigger at all, and there must not be: this + // workflow PUBLISHES the `eql-bindings` crate to crates.io over OIDC. A + // pull_request copy would be a publishing job reachable from a fork branch. + // So there is no second list to drift from. The direction that would hurt + // here is not a narrow filter but a MISSING run — a release that moves the + // crate and never triggers — and that is unreachable by construction: + // `changeset version` runs scripts/sync-lockstep-versions.mjs, which + // rewrites packages/eql/crates/eql-bindings/Cargo.toml and the SQL assets + // beside it, so every version-moving commit touches `packages/eql/**`. + // The list itself is kept honest by eql-workflow-filters.test.mjs. + 'push is the only trigger; a pull_request copy would make a crates.io publisher reachable from a fork', + ], [ '.github/workflows/tests-rust.yml', // `push:` is `branches: [main]` with no `paths:`, so every push to main runs diff --git a/scripts/__tests__/workflow-publish-permissions.test.mjs b/scripts/__tests__/workflow-publish-permissions.test.mjs index cd4decc8b..119ff296d 100644 --- a/scripts/__tests__/workflow-publish-permissions.test.mjs +++ b/scripts/__tests__/workflow-publish-permissions.test.mjs @@ -46,6 +46,45 @@ const OIDC_JOBS = [ '.github/workflows/release.yml / publish-ffi', // `changeset publish` for the JS packages, plus the Version Packages PR. '.github/workflows/release.yml / release', + // The EQL prerelease path publishes @cipherstash/eql directly rather than + // through changesets, so it mints its own npm token. + '.github/workflows/release.yml / prerelease-eql-npm', + // release-plz publishes the eql-bindings crate to crates.io, which uses the + // same OIDC token exchange. A DIFFERENT registry, and the reason this list + // could not stay a two-line one: the workflow filename is bound at crates.io + // rather than npm, but the scope is the same scope. + '.github/workflows/release-plz.yml / release', +] + +/** + * The jobs in a publishing workflow that may hold ANY writable scope. A + * superset of `OIDC_JOBS`, asserted as such below. + * + * A second list because `OIDC_JOBS` was doing double duty as the write list — + * fine while the only writing jobs were the two that publish, and not + * expressible once `release.yml` grew jobs that create a release, dispatch a + * workflow or move a branch ref. Those need a writable scope and must not be + * able to mint a publish token; adding them to `OIDC_JOBS` would have said they + * could. + */ +const REPO_WRITE_JOBS = [ + ...OIDC_JOBS, + // contents: write — creates the eql- tag and GitHub release, and + // uploads the SQL bundle to it. Reached through a job-level `uses:`, whose + // grant is the CEILING for the reusable workflow, so this is also what + // constrains `_build-eql-sql.yml`. + '.github/workflows/release.yml / eql-sql', + // contents: write — attaches the docs bundle to the release above. + '.github/workflows/release.yml / eql-docs', + // actions: write — `gh workflow run release-postgres-eql-image.yml`. + '.github/workflows/release.yml / eql-image', + // contents: write — the prerelease halves of the two above. + '.github/workflows/release.yml / prerelease-eql-sql', + '.github/workflows/release.yml / prerelease-eql-docs', + // actions: write to dispatch release-plz.yml, contents: write to pin the + // release/eql- branch it must be dispatched against (release-plz + // refuses a detached HEAD). + '.github/workflows/release.yml / prerelease-eql-crate', ] /** @@ -107,6 +146,9 @@ const workflows = workflowFiles().map((file) => { /** Is this the ` / ` of a job sanctioned to publish? */ const sanctioned = (file, name) => OIDC_JOBS.includes(`${file} / ${name}`) +/** …and of a job sanctioned to hold a writable scope at all? */ +const mayWrite = (file, name) => REPO_WRITE_JOBS.includes(`${file} / ${name}`) + describe('supply chain — a publishing workflow grants OIDC per job', () => { it('discovers the jobs that hold id-token: write', () => { // The guard on the scan. Every check below is "for each workflow that mints @@ -156,7 +198,7 @@ describe('supply chain — a publishing workflow grants OIDC per job', () => { ) .flatMap(({ file, workflowLevel, jobs }) => jobs - .filter(([name]) => !sanctioned(file, name)) + .filter(([name]) => !mayWrite(file, name)) .flatMap(([name, job]) => { const writes = writable(effective(job, workflowLevel)) return writes.length @@ -166,10 +208,22 @@ describe('supply chain — a publishing workflow grants OIDC per job', () => { ) expect( offenders, - 'A job in a publishing workflow that does not publish must not be able to write to the repository.', + 'A job in a publishing workflow that does not publish must not be able to write to the repository. If it genuinely needs a writable scope and must NOT be able to publish, add it to REPO_WRITE_JOBS with the scope and the reason — not to OIDC_JOBS.', ).toEqual([]) }) + it('never sanctions a write without sanctioning it as a write', () => { + // The one way the split above could go wrong: a publisher added to + // `OIDC_JOBS` and not carried into `REPO_WRITE_JOBS`. It is spelled as a + // spread today, so this cannot fail — which is the point. It fails the day + // somebody writes the two lists out separately, before the third check + // starts reporting a publisher as an offender. + const missing = OIDC_JOBS.filter( + (entry) => !REPO_WRITE_JOBS.includes(entry), + ) + expect(missing).toEqual([]) + }) + it('declares workflow-level permissions in a publishing workflow', () => { // Absent is not read-only: with no `permissions:` key at all, jobs fall back // to the REPOSITORY default, which is settings-controlled and outside this diff --git a/scripts/eql-pipeline-armed.mjs b/scripts/eql-pipeline-armed.mjs new file mode 100644 index 000000000..9ae50728a --- /dev/null +++ b/scripts/eql-pipeline-armed.mjs @@ -0,0 +1,60 @@ +/** + * May this repository publish the EQL release line? + * + * The pipeline was built here while `@cipherstash/eql` is still published from + * `cipherstash/encrypt-query-language`, so it must reach no registry until the + * Phase-5 cutover. That is a state, and it needs a switch. + * + * The switch is DERIVED, not flipped. `FROZEN_PUBLISHERS` in + * `scripts/release-gate.mjs` is the single record of "this package lives here + * and is published elsewhere", and the cutover has to delete its entry because + * the release gate blocks every release until it does. Deleting it arms this + * pipeline. A hand-flipped flag would have nothing forcing it: the cutover + * would repoint the registries, forget the flag, and the pipeline would stay + * inert — publishing an npm package with no SQL release, docs or crate. + * + * Why not reuse the release gate: it answers a REGISTRY question about npm, and + * `release-plz.yml` publishes a CRATE on its own trigger. Keying that on the npm + * answer would also race `release.yml` on the very push that releases a version. + * This asks a question with no registry and no clock in it. + */ +import { appendFileSync } from 'node:fs' +import process from 'node:process' +import { fileURLToPath } from 'node:url' +import { FROZEN_PUBLISHERS } from './release-gate.mjs' + +/** The package whose publisher decides whether the whole line is armed. */ +export const EQL_PACKAGE = '@cipherstash/eql' + +/** + * `true` when this repository may publish the EQL release line. + * + * The map is a parameter so the tests can drive both states — the armed one + * included, rather than exercising it for the first time at the cutover. + */ +export function eqlPipelineArmed(frozen = FROZEN_PUBLISHERS) { + return !frozen.has(EQL_PACKAGE) +} + +/** Why it is not armed, or `null`. Taken from the map, so it cannot drift. */ +export function frozenReason(frozen = FROZEN_PUBLISHERS) { + return frozen.get(EQL_PACKAGE) ?? null +} + +function main() { + const armed = eqlPipelineArmed() + + console.log( + armed + ? `${EQL_PACKAGE} is published from this repository — the EQL release pipeline is ARMED.` + : `${EQL_PACKAGE} is a frozen publisher — the EQL release pipeline is INERT.\n ${frozenReason()}`, + ) + + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `armed=${armed}\n`) + } +} + +// Importable without running, so the tests read the exports without writing to +// GITHUB_OUTPUT. +if (process.argv[1] === fileURLToPath(import.meta.url)) main() diff --git a/scripts/lint-no-eql-registry-pins.mjs b/scripts/lint-no-eql-registry-pins.mjs index c82749013..aacc6a669 100644 --- a/scripts/lint-no-eql-registry-pins.mjs +++ b/scripts/lint-no-eql-registry-pins.mjs @@ -104,9 +104,19 @@ const WORKSPACE_FILE = 'pnpm-workspace.yaml' * `target/package`, and every transitive dependency under `node_modules` — * which name their own dependencies by registry version, correctly. Scanning * them turns this linter into a permanent false alarm. + * + * `.claude` is the same argument one tool along. Claude Code puts agent + * worktrees under `.claude/worktrees/`, and a worktree is a FULL checkout of + * some other commit — including ones predating the in-tree flip, which still + * pin `@cipherstash/eql` by registry version and were correct when they were + * written. Descending into them reported 31 offenders and exited 1 on a clean + * tree. It is excluded through `.git/info/exclude`, which is local and travels + * to no other checkout, so `git check-ignore` is not a substitute for naming + * it here. */ const SKIP_DIRS = new Set([ '.git', + '.claude', 'node_modules', 'target', 'dist', diff --git a/scripts/lint-no-workflow-caching.mjs b/scripts/lint-no-workflow-caching.mjs index 167af61c7..b8d280523 100644 --- a/scripts/lint-no-workflow-caching.mjs +++ b/scripts/lint-no-workflow-caching.mjs @@ -17,6 +17,18 @@ const TARGETS = process.argv.slice(2).length // calls it too — cannot become a way to build these artifacts under // different rules. '.github/workflows/_build-ffi-artifacts.yml', + // The EQL release line. The two reusables are reached from release.yml + // anyway and named for the same reason `_build-ffi-artifacts.yml` is: a + // second caller must not become a way to build them under other rules. + // + // The other two are reached from NOWHERE — release.yml starts them with + // `gh workflow run`, which no traversal can follow — so naming them here + // is the only thing that brings them into scope. One publishes a crate + // over OIDC, the other pushes an image to GHCR. + '.github/workflows/_build-eql-sql.yml', + '.github/workflows/_build-eql-docs.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', '.github/workflows/tests-supply-chain.yml', ] @@ -155,6 +167,44 @@ const AUDITED_ACTIONS = new Map([ // fires only on a truthy value, which is how a mise-action step carrying no // `cache:` passed this gate. SHA-pinned at every call site. ['jdx/mise-action', { cacheInput: 'cache' }], + + // ---- The EQL release line ------------------------------------------- + // + // Creates and attaches the eql- GitHub release. Read at the pinned + // v2: release metadata and file globs, uploaded through the releases API. + ['softprops/action-gh-release', { cacheInput: null }], + // Imports the GPG key release-plz signs with. Read at v7: key material, git + // config and fingerprint inputs only. + ['crazy-max/ghaction-import-gpg', { cacheInput: null }], + // Publishes the eql-bindings crate. No cache input — AND IT IS A COMPOSITE + // reaching three published actions of its own (taiki-e/install-action, + // cargo-bins/cargo-binstall, release-plz/git-config) that this gate cannot + // read; it opens local `./` composites only. So this is an audit of the + // pinned version, not a standing property: re-read it when the pin moves. + // Read at 2eb1d8bc — all three download binaries, none touches the cache. + ['release-plz/action', { cacheInput: null }], + + // ---- Docker, for the postgres-eql image ------------------------------ + // + // The plan called all three "registry-backed rather than + // GitHub-Actions-cache". Reading the pinned manifests, that is true of one. + // + // `cache-image` DEFAULTS TO TRUE — binfmt image in the Actions cache. + ['docker/setup-qemu-action', { cacheInput: 'cache-image' }], + // `cache-binary` DEFAULTS TO TRUE — the buildx binary, same way. + ['docker/setup-buildx-action', { cacheInput: 'cache-binary' }], + // GHCR auth. No cache, no cache input. + ['docker/login-action', { cacheInput: null }], + // The one the plan described, and the only `forbiddenInputs` entry. It has no + // boolean to turn caching off because it has none of its own: buildx caches + // what `cache-from` / `cache-to` ask for. Usually registry-backed, which is + // outside this gate — but `type=gha` is the Actions cache, restored AND + // written inside the job that pushes the image. Forbidden outright; adding + // one back means deciding here which backend it names. + [ + 'docker/build-push-action', + { cacheInput: null, forbiddenInputs: ['cache-from', 'cache-to'] }, + ], ]) // A secondary, deliberately over-broad read of the action's name. It is NOT the @@ -307,12 +357,28 @@ function checkStep(step, at, bodyAudited = false) { // from a parallel list of per-action rules. An audited action that caches is // an additional constraint on it, not a substitute for being audited. const path = actionPath(uses) - const cacheInput = AUDITED_ACTIONS.get(path)?.cacheInput + const audited = AUDITED_ACTIONS.get(path) + const cacheInput = audited?.cacheInput if (cacheInput) { const reason = explicitFalseReason(step, cacheInput) if (reason) offenders.push(`${at}: ${path} ${reason}`) } + // The same audit for an action whose caching is opted INTO rather than + // defaulted on: nothing to set to `false`, so the input must be absent. + // Presence is the finding whatever the value — the value is a backend name, + // and telling `type=gha` from `type=registry` would mean owning a parser for + // buildx cache specs. Naming one is the moment to record the decision above. + for (const forbidden of audited?.forbiddenInputs ?? []) { + if (step?.with && Object.hasOwn(step.with, forbidden)) { + offenders.push( + `${at}: ${path} must not set \`${forbidden}\` — it can name the GitHub ` + + 'Actions cache (`type=gha`), and this action has no input that turns ' + + 'caching off. See its AUDITED_ACTIONS entry.', + ) + } + } + // One verdict per `uses:`, most specific first — a step reported twice reads // as two problems and gets fixed once. if (CACHE_ACTION.test(uses)) {