From cab02010878f718a488a1ca48a814fb246e9fbd5 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 24 Aug 2026 12:35:22 +1000 Subject: [PATCH 1/5] ci(eql): port the EQL release pipeline to the repository root, inert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EQL subtree arrived with eleven files under `packages/eql/.github/`, a directory GitHub never reads. Seven of them were workflows, and between them they are the entire publishing pipeline for `@cipherstash/eql`: the npm package, the `eql-bindings` crate, the SQL bundle, the docs bundle and the `postgres-eql` image, all of which ship at one version. None of it executed. This ports the lot and keeps it inert, so it can be reviewed and dry-run before trusted publishing is repointed at this repository. Inertness is a derived switch rather than a flag. `scripts/eql-pipeline-armed.mjs` reads `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs` — the map that already records "this package lives here and is published elsewhere" — and every job that publishes an EQL artefact is gated on its answer. The cutover has to delete that entry (the release gate refuses every release until it does), so deleting it is what arms the pipeline; there is no second flag to forget, and forgetting one would fail silently in the direction that publishes an npm package with no SQL release, no docs and no crate. Three paths had to be rewritten because the subtree root is not the package root here: the npm manifest is two levels down at `packages/eql/packages/eql`, mise config is only found from `packages/eql`, and `release-plz/action` needs explicit `manifest_path` / `config` inputs because a composite action's steps do not inherit the caller's `defaults.run.working-directory`. Three of this repo's guards had to grow rather than be exempted, and two real defects came out of them: the prerelease publish job was missing `node-gyp` before its install, and built with `pnpm --filter` rather than through turbo. Claude-Session: https://claude.ai/code/session_01T26DhFPkLfN3qPFDq2ttKw --- .github/workflows/_build-eql-docs.yml | 101 ++++ .github/workflows/_build-eql-sql.yml | 147 +++++ .github/workflows/lint-release.yml | 59 +- .github/workflows/rebuild-docs.yml | 22 + .github/workflows/release-plz.yml | 189 ++++++ .../workflows/release-postgres-eql-image.yml | 251 ++++++++ .github/workflows/release.yml | 558 +++++++++++++++++- AGENTS.md | 79 ++- SECURITY.md | 24 +- .../2026-08-13-eql-monorepo-absorption.md | 18 +- .../.github/ISSUE_TEMPLATE/docs-feedback.yml | 41 -- packages/eql/.github/actionlint.yaml | 4 - packages/eql/.github/release.yml | 5 - packages/eql/.github/workflows/README.md | 228 ------- .../eql/.github/workflows/_build-docs.yml | 85 --- packages/eql/.github/workflows/_build-sql.yml | 122 ---- .../eql/.github/workflows/lint-release.yml | 79 --- .../eql/.github/workflows/rebuild-docs.yml | 20 - .../eql/.github/workflows/release-plz.yml | 113 ---- .../workflows/release-postgres-eql-image.yml | 169 ------ packages/eql/.github/workflows/release.yml | 426 ------------- packages/eql/release-plz.toml | 31 +- scripts/__tests__/eql-pipeline-armed.test.mjs | 161 +++++ scripts/__tests__/eql-suite-ci.test.mjs | 213 +++++-- .../lint-no-workflow-caching.test.mjs | 8 + .../workflow-dispatch-job-conditions.test.mjs | 86 ++- .../workflow-publish-permissions.test.mjs | 69 ++- scripts/eql-pipeline-armed.mjs | 93 +++ scripts/lint-no-workflow-caching.mjs | 89 ++- 29 files changed, 2079 insertions(+), 1411 deletions(-) create mode 100644 .github/workflows/_build-eql-docs.yml create mode 100644 .github/workflows/_build-eql-sql.yml create mode 100644 .github/workflows/release-plz.yml create mode 100644 .github/workflows/release-postgres-eql-image.yml delete mode 100644 packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml delete mode 100644 packages/eql/.github/actionlint.yaml delete mode 100644 packages/eql/.github/release.yml delete mode 100644 packages/eql/.github/workflows/README.md delete mode 100644 packages/eql/.github/workflows/_build-docs.yml delete mode 100644 packages/eql/.github/workflows/_build-sql.yml delete mode 100644 packages/eql/.github/workflows/lint-release.yml delete mode 100644 packages/eql/.github/workflows/rebuild-docs.yml delete mode 100644 packages/eql/.github/workflows/release-plz.yml delete mode 100644 packages/eql/.github/workflows/release-postgres-eql-image.yml delete mode 100644 packages/eql/.github/workflows/release.yml create mode 100644 scripts/__tests__/eql-pipeline-armed.test.mjs create mode 100644 scripts/eql-pipeline-armed.mjs diff --git a/.github/workflows/_build-eql-docs.yml b/.github/workflows/_build-eql-docs.yml new file mode 100644 index 000000000..ae5b5e325 --- /dev/null +++ b/.github/workflows/_build-eql-docs.yml @@ -0,0 +1,101 @@ +name: "Build EQL docs (reusable)" + +# Reusable docs build + attach for the EQL release line. Called inline by +# `release.yml` on BOTH the production and the prerelease path. The target +# release already exists by the time this runs: `_build-eql-sql.yml` creates it +# earlier in the same run, which is why both callers declare it as a `needs:`. +# +# Ported from `packages/eql/.github/workflows/_build-docs.yml` — see +# `_build-eql-sql.yml`'s header for the rename, the two `packages/eql` +# rewrites, and why `defaults.run.shell: bash {0}` was dropped. +# +# THIS PIPELINE HAS NEVER EXECUTED IN THIS REPOSITORY. `docs:generate:json`'s +# only caller was the unported copy of this file, which is why it carried a +# written exemption in `scripts/__tests__/eql-suite-ci.test.mjs` until this +# workflow landed. The doxygen half is the part to watch: `docs:generate` shells +# out to a `doxygen` binary that mise does not provide (`packages/eql/mise.toml` +# pins `python = "3.13"` under `[tools]`, and nothing else), so the apt install +# below is load-bearing and is the only thing standing between this job and a +# `doxygen: command not found` on the first real release. +on: + workflow_call: + inputs: + ref: + description: "Git ref/SHA to build docs from. Empty -> default checkout (github.sha)." + required: false + type: string + default: "" + tag: + description: "Full release tag. Empty -> build only, no attach." + required: false + type: string + default: "" + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + MISE_VERBOSE: "1" + +defaults: + run: + working-directory: packages/eql + +permissions: + contents: write + +jobs: + publish-docs: + name: Build and publish documentation + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + # Reached only from `release.yml`; see `_build-eql-sql.yml` for why a + # publish path restores no cache. Upstream had `cache: true`. + cache: false + working_directory: packages/eql + + - name: Install Doxygen + # Not from mise: `packages/eql/mise.toml` does not pin doxygen, and + # `tasks/docs/generate.sh` invokes the binary directly. + run: | + sudo apt-get update + sudo apt-get install -y doxygen + + - name: Generate documentation + env: + TAG: ${{ inputs.tag }} + run: | + mise run docs:generate + mise run docs:generate:markdown -- "${TAG}" + mise run docs:generate:json -- "${TAG}" + + - name: Package documentation + env: + TAG: ${{ inputs.tag }} + run: mise run docs:package "${TAG}" + + - name: Upload documentation artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-docs + path: | + packages/eql/release/eql-docs-*.zip + packages/eql/release/eql-docs-*.tar.gz + + - name: Publish documentation to release + if: ${{ inputs.tag != '' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag }} + files: | + packages/eql/release/eql-docs-*.zip + packages/eql/release/eql-docs-*.tar.gz diff --git a/.github/workflows/_build-eql-sql.yml b/.github/workflows/_build-eql-sql.yml new file mode 100644 index 000000000..bbe6b0799 --- /dev/null +++ b/.github/workflows/_build-eql-sql.yml @@ -0,0 +1,147 @@ +name: "Build EQL SQL (reusable)" + +# Reusable SQL build + attach for the EQL release line. Called inline by +# `release.yml` on BOTH the production and the prerelease path, so an EQL SQL +# release has exactly one build path regardless of which one produced it. +# +# Ported from `packages/eql/.github/workflows/_build-sql.yml`, which GitHub +# never read — see the header of `test-eql.yml` for the same story about the +# test suite. Four things changed on the way in, each silent when wrong: +# +# * RENAMED. Upstream this file was `_build-sql.yml`, in a repository whose +# only artefact was SQL. Here `_build-ffi-artifacts.yml` is the sibling and +# "SQL" names nothing on its own, so the file follows the `*-eql.yml` +# convention `test-eql.yml` / `bench-eql.yml` / `macro-expand-eql.yml` +# already set. Nothing binds to the filename: it is reachable only through +# a job-level `uses:` in `release.yml`. +# * `defaults.run.working-directory: packages/eql` — `mise run build` and +# `release/*.sql` are written against the EQL root, which is one level down +# here. Without it they resolve against the monorepo root. +# * `working_directory:` on the mise step — `defaults.run` does not reach a +# `uses:` step, and mise reads config from the current directory and its +# PARENTS only, so an action running at the repo root never sees +# `packages/eql/mise.toml`. It installs nothing and leaves the config +# untrusted, and the first `mise run` fails with "Config files … are not +# trusted" — which reads as a broken toolchain rather than a wrong +# directory. +# * artifact and release `path:` / `files:` values are workspace-root +# relative and `defaults.run` does not touch them either, so they carry the +# `packages/eql/` prefix explicitly. +# +# `defaults.run.shell` is deliberately NOT overridden, where the imported file +# carried `bash {0}`. GitHub's default `bash` runs with `-eo pipefail`; `bash +# {0}` gives exactly the flags asked for, which is none of them, and a failing +# first line of a multi-command block is then discarded. See +# `scripts/__tests__/workflow-run-fail-fast.test.mjs`. +# +# THE MULTITUDES DEPLOY PING WAS DROPPED. Upstream ended this job with a +# `continue-on-error` curl to `api.developer.multitudes.co` authenticated with +# `MULTITUDES_ACCESS_TOKEN`. That secret does not exist in this repository +# (`gh secret list`), so the step would have failed on every production release +# and been swallowed by its own `continue-on-error` — indistinguishable from +# one that worked. Deploy tracking for `cipherstash/stack` is a repository-wide +# decision, not something an EQL SQL release should carry alone; if it is +# wanted, it arrives with the secret. + +on: + workflow_call: + inputs: + ref: + description: "Git ref/SHA to build from. Empty -> default checkout (github.sha)." + required: false + type: string + default: "" + tag: + description: "Full release tag (e.g. eql-3.0.6). Empty -> DEV build, no attach." + required: false + type: string + default: "" + attach: + description: "Attach the built .sql artefacts to a GitHub Release." + required: false + type: boolean + default: false + target_commitish: + description: "Non-empty -> create a release at this commit; empty -> attach to the existing release named by tag." + required: false + type: string + default: "" + prerelease: + description: "Mark the created release as a prerelease (create path only)." + required: false + type: boolean + default: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + MISE_VERBOSE: "1" + +defaults: + run: + working-directory: packages/eql + +permissions: + contents: write + +jobs: + build: + name: Build EQL + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + # `cache: false` IS NOT THE DEFAULT — mise-action caches unless told + # otherwise, and `scripts/lint-no-workflow-caching.mjs` forbids a + # GitHub Actions cache restore anywhere an artefact gets published. + # This workflow is reached only from `release.yml`, the job that holds + # the npm publishing credential, so a poisoned toolchain entry here + # would compile the SQL that ships. Upstream had `cache: true`. + cache: false + working_directory: packages/eql + + - name: Build EQL release + # Strip `eql-` so `eql_v3.version()` reports bare semver. An empty TAG + # intentionally falls through `tasks/build.sh`'s `${usage_version:-DEV}` + # default. + env: + TAG: ${{ inputs.tag }} + run: mise run build --version "${TAG#eql-}" + + - name: Upload EQL artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-release + path: | + packages/eql/release/cipherstash-encrypt.sql + packages/eql/release/cipherstash-encrypt-uninstall.sql + + - name: Attach artefacts to existing release + if: ${{ inputs.attach && inputs.target_commitish == '' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag }} + files: | + packages/eql/release/cipherstash-encrypt.sql + packages/eql/release/cipherstash-encrypt-uninstall.sql + + - name: Create release at commit + if: ${{ inputs.attach && inputs.target_commitish != '' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag }} + target_commitish: ${{ inputs.target_commitish }} + prerelease: ${{ inputs.prerelease }} + name: ${{ inputs.tag }} + body: "The standalone eql_v3 SQL surface. See packages/eql/CHANGELOG.md." + files: | + packages/eql/release/cipherstash-encrypt.sql + packages/eql/release/cipherstash-encrypt-uninstall.sql diff --git a/.github/workflows/lint-release.yml b/.github/workflows/lint-release.yml index 26c4d147b..c4b1f4d35 100644 --- a/.github/workflows/lint-release.yml +++ b/.github/workflows/lint-release.yml @@ -10,27 +10,40 @@ name: Lint release tooling # shellcheck over every `run:` block — which is why the workflows it gates # avoid `sed`-into-`export` (SC2001, SC2155), `ls | wc -l` (SC2012), and # dollar-braces or backticks inside single-quoted `node -e` arguments (SC2016). +# +# THE EQL RELEASE WORKFLOWS ARE IN SCOPE, and that is the whole reason their +# `actionlint` cleanliness is a property rather than a coincidence. They were +# ported from `packages/eql/.github/`, where an EQL-local `lint-release.yml` +# linted them and — like everything else in that directory — ran on nothing. +# Its two unique checks came with them: `shellcheck` over the +# `prepare-bindings-assets` wrapper and that wrapper's own bash unit test, both +# of which ran NOWHERE in this repository until this file named them. on: pull_request: - # Exactly what the job reads: the four workflows actionlint is pointed at, - # plus the config it resolves the Blacksmith label from. The filter also - # named scripts/release-gate.mjs, scripts/ffi-release-matrix.mjs, - # scripts/lint-no-workflow-caching.mjs and package.json — none of which this - # job looks at, since it deliberately does not run `test:scripts` (see the - # job comment). Editing one booted a runner and downloaded a Go binary to - # lint four unchanged files, which is how a job trains reviewers to ignore - # it. `tests.yml`'s `lint` job runs `test:scripts` unfiltered on every PR, - # so those four are already covered. + # Exactly what the job reads: the workflows actionlint is pointed at, the + # config it resolves the Blacksmith label from, and the two shell scripts + # the last two steps run. The filter also named scripts/release-gate.mjs, + # scripts/ffi-release-matrix.mjs, scripts/lint-no-workflow-caching.mjs and + # package.json — none of which this job looks at, since it deliberately does + # not run `test:scripts` (see the job comment). Editing one booted a runner + # and downloaded a Go binary to lint four unchanged files, which is how a + # job trains reviewers to ignore it. `tests.yml`'s `lint` job runs + # `test:scripts` unfiltered on every PR, so those four are already covered. # - # `lint-release-scope.test.mjs` asserts this list and the actionlint - # argument list below stay the same set. + # `lint-release-scope.test.mjs` asserts the `.github/workflows/` half of + # this list and the actionlint argument list below stay the same set. paths: - .github/workflows/release.yml - .github/workflows/_build-ffi-artifacts.yml - .github/workflows/ffi-preflight.yml + - .github/workflows/_build-eql-sql.yml + - .github/workflows/_build-eql-docs.yml + - .github/workflows/release-plz.yml + - .github/workflows/release-postgres-eql-image.yml - .github/workflows/lint-release.yml - .github/actionlint.yaml + - packages/eql/tasks/release/*.sh workflow_dispatch: {} permissions: @@ -46,15 +59,15 @@ concurrency: jobs: # No pnpm, no Node, no install: actionlint is a downloaded Go binary and - # shellcheck ships in the runner image, so this job is a checkout and one - # command. + # shellcheck ships in the runner image, so this job is a checkout and a few + # commands. # # It deliberately does NOT also run `test:scripts` or `lint:workflow-cache`. # `tests.yml`'s `lint` job runs `test:scripts` on every pull request with no # path filter, and `tests-supply-chain.yml` runs both — so a copy here would # be the second and third run of the same checks on any PR touching the # release machinery, each behind its own uncached full-workspace install. What - # this workflow uniquely has is actionlint. + # this workflow uniquely has is actionlint, and the two shell checks below. lint: name: actionlint (release workflows) runs-on: ubuntu-latest @@ -81,5 +94,23 @@ jobs: .github/workflows/release.yml \ .github/workflows/_build-ffi-artifacts.yml \ .github/workflows/ffi-preflight.yml \ + .github/workflows/_build-eql-sql.yml \ + .github/workflows/_build-eql-docs.yml \ + .github/workflows/release-plz.yml \ + .github/workflows/release-postgres-eql-image.yml \ .github/workflows/lint-release.yml + # The EQL half. `prepare-bindings-assets.sh` is what stamps the SQL bundle + # and rewrites the four release manifests during `pnpm run version`, so a + # shell mistake in it lands mid-release with every manifest already + # rewritten. Neither of these ran anywhere in this repository before the + # release port. + - name: shellcheck (EQL release wrappers) + run: | + set -euo pipefail + shellcheck \ + packages/eql/tasks/release/prepare-bindings-assets.sh \ + packages/eql/tasks/release/prepare-bindings-assets.test.sh + + - name: prepare-bindings-assets validation unit test + run: bash packages/eql/tasks/release/prepare-bindings-assets.test.sh diff --git a/.github/workflows/rebuild-docs.yml b/.github/workflows/rebuild-docs.yml index 858732977..78795b5fb 100644 --- a/.github/workflows/rebuild-docs.yml +++ b/.github/workflows/rebuild-docs.yml @@ -1,14 +1,36 @@ name: Rebuild Docs +# Fires the docs-site rebuild webhook when a release tag lands. +# +# TWO TAG FAMILIES, ONE WEBHOOK. This file used to watch only the changesets tag +# for `@cipherstash/stack`. The EQL port added the second family: the imported +# copy at `packages/eql/.github/workflows/rebuild-docs.yml` watched `eql-*` and +# ran nowhere, because GitHub reads workflows from the repository root alone. +# Merging them rather than keeping two files was the obvious call — the job is +# one `curl` against one secret, and nothing binds to either filename. +# +# `!eql-bindings-*` IS PART OF THE RELEASE IDENTITY, not a tidy-up. release-plz +# tags the Rust crate `eql-bindings-v` (its default workspace tag), and +# a crate release must not fire the docs rebuild — nor the SQL build or the +# Postgres image, which is why `packages/eql/release-plz.toml` chose a tag shape +# the `eql-*` automation does not match. GitHub applies these patterns in order: +# a ref matching a negative pattern AFTER a positive one is excluded, so +# `eql-bindings-v3.0.6` matches `eql-*` and is then dropped. on: push: tags: - '@cipherstash/stack@*' + - 'eql-*' + - '!eql-bindings-*' + +permissions: + contents: read jobs: trigger-docs-rebuild: name: Trigger Docs Rebuild runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Send webhook env: diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml new file mode 100644 index 000000000..33822e460 --- /dev/null +++ b/.github/workflows/release-plz.yml @@ -0,0 +1,189 @@ +name: "Release eql-bindings (crates.io)" + +# Publishes the `eql-bindings` crate to crates.io via release-plz (crates.io +# Trusted Publishing over OIDC — no long-lived token; GPG-signed release +# commits and tags). +# +# THE FILENAME IS LOAD-BEARING AND MUST NOT CHANGE. crates.io Trusted +# Publishing binds a crate to a repository AND a workflow filename, exactly as +# npm's does. `_build-sql.yml` and `_build-docs.yml` could be renamed on the way +# in (see `_build-eql-sql.yml`); this one could not. +# +# PUBLISH-ONLY. Crate versioning is owned by CHANGESETS, not release-plz. The +# Version Packages PR bumps `packages/eql/crates/eql-bindings/Cargo.toml` to the +# lockstep version V (`scripts/sync-lockstep-versions.mjs`, driven by the root +# `version` script) alongside the npm package. This workflow publishes the +# COMMITTED version — a no-op when V is already on crates.io — and tags +# `eql-bindings-v`. There is deliberately no release-plz `release-pr` job: a +# release-plz version PR would fight the changesets one. +# +# THE `eql-bindings-v*` TAG SHAPE IS PART OF THE RELEASE IDENTITY. It is +# release-plz's default workspace tag, and it deliberately does NOT match the +# `eql-` tags the SQL surface releases under. `rebuild-docs.yml` carries +# the matching `!eql-bindings-*` exclusion, so a crate release never fires the +# docs-site rebuild, the SQL build or the Postgres image. +# +# ## Inert until the cutover +# +# `@cipherstash/eql` is still published from `cipherstash/encrypt-query-language` +# — including this crate, whose crates.io Trusted Publishing configuration names +# that repository and this filename. So the `release` job below is gated on +# `scripts/eql-pipeline-armed.mjs`, which reads `FROZEN_PUBLISHERS` in +# `scripts/release-gate.mjs`. Deleting that entry at the Phase-5 cutover is what +# arms this workflow; there is no separate flag to remember. Read that script's +# header for why the switch is derived rather than flipped, and why the npm +# release gate could not be reused here (crates.io is not a registry it models, +# and keying on the npm answer would race `release.yml` on the very push that +# releases a version). +# +# ## Before the cutover arms this, two things must exist +# +# 1. `GPG_PRIVATE_KEY` in this repository's secrets. It does not exist today +# (`gh secret list`), and the import step below fails without it. +# 2. A crates.io Trusted Publishing entry on the `eql-bindings` crate naming +# Repository `cipherstash/stack` and Workflow `release-plz.yml`. +# +# crates.io auth is OIDC (`id-token: write`), so there is NO +# CARGO_REGISTRY_TOKEN. GitHub PR/tag operations use the default GITHUB_TOKEN. + +# READ-ONLY BY DEFAULT; the one job that publishes escalates for itself. Same +# rule and same reason as `release.yml`: a workflow-level `id-token: write` is a +# default rather than a ceiling, so every job in a file registered as a trusted +# publisher would be able to mint a token the registry accepts. Enforced by +# `scripts/__tests__/workflow-publish-permissions.test.mjs`. +permissions: + contents: read + +on: + push: + branches: + - main + workflow_dispatch: {} + +# Static group: push-to-main and workflow_dispatch share it so two release-plz +# invocations never race a tag or a publish. Never cancel — a cancelled release +# could leave a half-published state. +concurrency: + group: release-plz + cancel-in-progress: false + +jobs: + # WHETHER THIS REPOSITORY MAY PUBLISH THE CRATE AT ALL. A checkout and one + # `node` call reading a Map — no registry lookup, so it cannot disagree with + # itself between runs or race the npm publish happening in `release.yml` on + # the same push. + eql-armed: + name: Is the EQL release line armed? + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + armed: ${{ steps.armed.outputs.armed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + # No cache anywhere in a publishing workflow — see `release.yml`. + package-manager-cache: false + + # No pnpm, no install: the script imports node builtins and + # `scripts/release-gate.mjs`, which does the same. + - name: Read the publisher switch + id: armed + run: node scripts/eql-pipeline-armed.mjs + + release: + name: "Release" + needs: [eql-armed] + if: needs.eql-armed.outputs.armed == 'true' + # GitHub-hosted, not Blacksmith. crates.io Trusted Publishing exchanges a + # GitHub OIDC token, and the equivalent npm path rejects self-hosted runners + # outright (E422); this repository's other publishing jobs are all + # `ubuntu-latest` for that reason and this one matches them rather than + # keeping upstream's Blacksmith label. + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # release-plz creates the eql-bindings-v tag and release + pull-requests: write # release-plz reads/links PRs when writing release notes + id-token: write # crates.io Trusted Publishing (OIDC) + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + # Sign release-plz's commits and tags with the shared release key. + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + git_user_signingkey: true + git_commit_gpgsign: true + git_tag_gpgsign: true + + # NO mise STEP, where upstream had one. Nothing in this job runs a mise + # task: the guard below is `curl` + `jq` + `grep` from the runner image, + # and `release-plz/action` needs only `cargo` and `git`, both of which + # `ubuntu-latest` ships. Installing mise here would have cost a third-party + # action inside the job that holds the crates.io credential, and it would + # have had to carry `add_shims_to_path: false` (the rule + # scripts/__tests__/workflow-mise-setup.test.mjs applies to every + # `id-token: write` job) — at which point mise's toolchain would not be on + # PATH and the step would be installing tools nothing could reach. + # + # The pinned toolchain is not what is lost. `packages/eql/mise.toml` pins + # `rust = { version = "latest" }`, so it never named a fixed compiler; the + # image's rustup gives a recent stable and `cargo publish` packages the + # committed tree either way. + - name: Verify bundled SQL matches the crate version + # The crate bundles its SQL via `include_str!` on COMMITTED files, and + # release-plz publishes the committed tree verbatim. Refuse to publish a + # crate whose bundled SQL was not prepared for this exact version — a + # hand-pinned Cargo.toml without `pnpm run version` would otherwise ship + # the DEV placeholder as its "exact SQL". Enforced only when this run + # would actually publish (the committed version is not on crates.io + # yet): routine no-op runs on main between releases carry the DEV + # placeholder legitimately. A crates.io API failure fails towards + # ENFORCING the guard, never towards skipping it. + working-directory: packages/eql + run: | + set -euo pipefail + cargo_version="$(grep -m1 '^version = ' crates/eql-bindings/Cargo.toml | cut -d'"' -f2)" + # crates.io rejects curl's default User-Agent with a 403. Without an + # explicit UA the request always fails, `|| true` swallows it, and the + # no-op short-circuit below is unreachable. + published="$(curl -fsSL --retry 3 -H "User-Agent: cipherstash-eql-release (https://github.com/cipherstash/stack)" "https://crates.io/api/v1/crates/eql-bindings/versions" | jq -r '.versions[].num' | grep -Fx "$cargo_version" || true)" + if [[ -n "$published" ]]; then + echo "eql-bindings@${cargo_version} is already on crates.io; this run is a publish no-op — skipping the bundled-SQL guard" + exit 0 + fi + manifest_version="$(jq -r '.eqlVersion' crates/eql-bindings/sql/release-manifest.json)" + if [[ "$manifest_version" != "$cargo_version" ]]; then + echo "::error::packages/eql/crates/eql-bindings/sql/release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml version ('$cargo_version') — the bundled SQL was not prepared for this release. Run 'pnpm run version' from the repository root (or 'mise run release:prepare_bindings_assets --version $cargo_version' from packages/eql) and commit the result." >&2 + exit 1 + fi + # `grep -q` on a FILE, not a pipeline: there is no upstream writer to + # take SIGPIPE. See scripts/__tests__/workflow-grep-q-pipelines.test.mjs. + if ! grep -q "eql_v3" crates/eql-bindings/sql/cipherstash-encrypt.sql; then + echo "::error::packages/eql/crates/eql-bindings/sql/cipherstash-encrypt.sql looks like the DEV placeholder — refusing to publish it as release SQL." >&2 + exit 1 + fi + + - name: Run release-plz release + uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5 + with: + command: release + # BOTH PATHS ARE REQUIRED HERE AND WERE ABSENT UPSTREAM, where the + # repository root WAS the Cargo workspace root. A composite action's + # steps do not inherit the caller's `defaults.run.working-directory`, + # so `working-directory:` on this step would do nothing — the paths + # have to be passed as inputs. Without them release-plz reads the + # monorepo root, finds no `[workspace]` naming `eql-bindings`, and + # publishes nothing while exiting 0. + manifest_path: packages/eql/Cargo.toml + config: packages/eql/release-plz.toml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-postgres-eql-image.yml b/.github/workflows/release-postgres-eql-image.yml new file mode 100644 index 000000000..7f1c53291 --- /dev/null +++ b/.github/workflows/release-postgres-eql-image.yml @@ -0,0 +1,251 @@ +name: "Release Postgres + EQL image" + +# Builds `ghcr.io/cipherstash/postgres-eql` — Postgres with the EQL SQL surface +# already installed — for PG 14 through 17, and pushes it to GHCR. +# +# Ported from `packages/eql/.github/workflows/release-postgres-eql-image.yml`. +# Every path it names moved one level down; see `_build-eql-sql.yml`'s header +# for the `defaults.run.working-directory` / `working_directory:` split and why +# `defaults.run.shell: bash {0}` was dropped. +# +# DISPATCH-ONLY, and that is deliberate rather than incidental. `release.yml` +# dispatches it on production finals with `update_floating_tags=true`. It is not +# `on: release`, because a release created by GITHUB_TOKEN does not trigger +# `on: release` fan-out — whereas a GITHUB_TOKEN `workflow_dispatch` does. +# +# ## Inert until the cutover +# +# The `eql-armed` job below reads `scripts/eql-pipeline-armed.mjs`, so a manual +# dispatch does nothing while `@cipherstash/eql` is still published from +# `cipherstash/encrypt-query-language`. `release.yml`'s dispatch is behind the +# same switch, so the two cannot disagree. Deleting the `FROZEN_PUBLISHERS` +# entry at the Phase-5 cutover arms both. +# +# The GHCR package itself is a second, independent gate that this workflow does +# not control: `ghcr.io/cipherstash/postgres-eql` is currently linked to +# `cipherstash/encrypt-query-language`, so this repository's GITHUB_TOKEN has no +# write access to it until the package's own repository access is updated. That +# has to happen with the cutover, and it is not something a workflow can assert +# ahead of time — the first armed run is what proves it. + +on: + workflow_dispatch: + inputs: + eql_version: + description: "EQL version to embed (e.g. 3.0.6). Used as the image tag and the SQL build version." + required: true + type: string + update_floating_tags: + description: "Also update :latest, :, and : floating tags. Off by default for test runs." + required: false + type: boolean + default: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + MISE_VERBOSE: "1" + IMAGE: ghcr.io/cipherstash/postgres-eql + +permissions: + contents: read + +jobs: + # See `release-plz.yml` for why the switch is derived from + # `FROZEN_PUBLISHERS` rather than being a flag somebody flips. + eql-armed: + name: Is the EQL release line armed? + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + armed: ${{ steps.armed.outputs.armed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + - name: Read the publisher switch + id: armed + run: node scripts/eql-pipeline-armed.mjs + + build-sql: + name: Build EQL SQL + needs: [eql-armed] + if: needs.eql-armed.outputs.armed == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 20 + defaults: + run: + working-directory: packages/eql + outputs: + # Used for image tags, e.g. "3.0.6" -> ghcr.io/.../postgres-eql:17-3.0.6 + eql_version: ${{ steps.ver.outputs.eql_version }} + # Passed to `mise run build --version`. Bare semver (no `eql-` prefix), + # matching the SQL release build (`_build-eql-sql.yml`), so + # `eql_v3.version()` in the image is byte-identical to the SQL release. + build_version: ${{ steps.ver.outputs.build_version }} + update_floating_tags: ${{ steps.ver.outputs.update_floating_tags }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + # This workflow pushes an image, so it is a publish path: no cache. + # Upstream had `cache: true`. Enforced by + # `scripts/lint-no-workflow-caching.mjs`. + cache: false + working_directory: packages/eql + + - name: Compute EQL version and tag policy + id: ver + # `eql_version` arrives as bare semver (`release.yml` sends + # `needs.release.outputs.eql_version`, e.g. "3.0.6"), used both as the + # image tag and, verbatim, as the `mise run build --version` value. + env: + INPUT_VERSION: ${{ inputs.eql_version }} + INPUT_FLOATING: ${{ inputs.update_floating_tags }} + run: | + { + echo "eql_version=${INPUT_VERSION}" + echo "build_version=${INPUT_VERSION}" + echo "update_floating_tags=${INPUT_FLOATING}" + } >> "$GITHUB_OUTPUT" + + - name: Build EQL release SQL + env: + BUILD_VERSION: ${{ steps.ver.outputs.build_version }} + run: mise run build --version "$BUILD_VERSION" + + - name: Upload EQL SQL artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-sql + # Workspace-root relative: `defaults.run` does not reach a `uses:` step. + path: packages/eql/release/cipherstash-encrypt.sql + + build-images: + name: Build postgres-eql image (PG ${{ matrix.pg_version }}) + needs: [eql-armed, build-sql] + if: needs.eql-armed.outputs.armed == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 30 + permissions: + contents: read + packages: write # push to ghcr.io + strategy: + fail-fast: false + matrix: + pg_version: ["14", "15", "16", "17"] + env: + EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} + UPDATE_FLOATING: ${{ needs.build-sql.outputs.update_floating_tags }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Download EQL SQL artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: eql-sql + # The Dockerfile `COPY`s the bundle from its own build context, which + # is `packages/eql/docker` below. + path: packages/eql/docker/ + + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + # `cache-image` DEFAULTS TO TRUE and stores the binfmt image in the + # GitHub Actions cache. This job pushes a published image, so it + # restores nothing. See scripts/lint-no-workflow-caching.mjs. + cache-image: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + # `cache-binary` DEFAULTS TO TRUE — same rule, same reason. + cache-binary: false + + - name: Login to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image tags + id: tags + # Always push the immutable :- tag. Only push the floating + # : tag on a release, or on a dispatch that opted in. + env: + PG_VERSION: ${{ matrix.pg_version }} + run: | + tags="${IMAGE}:${PG_VERSION}-${EQL_VERSION}" + if [[ "${UPDATE_FLOATING}" == "true" ]]; then + tags+=$'\n'"${IMAGE}:${PG_VERSION}" + fi + { + echo 'tags<> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + # NO `cache-from:` / `cache-to:`. Both are absent rather than disabled — + # this action has no boolean that turns caching off, and `type=gha` + # names the GitHub Actions cache directly. See the AUDITED_ACTIONS entry + # in scripts/lint-no-workflow-caching.mjs, which forbids the two inputs + # outright rather than asking for a `false` that does not exist. + with: + context: packages/eql/docker + file: packages/eql/docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + build-args: | + PG_VERSION=${{ matrix.pg_version }} + EQL_VERSION=${{ env.EQL_VERSION }} + tags: ${{ steps.tags.outputs.tags }} + + promote-latest: + name: Promote PG17 image to :latest and : + needs: [eql-armed, build-sql, build-images] + if: needs.build-sql.outputs.update_floating_tags == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 10 + permissions: + contents: read + packages: write # re-tag in ghcr.io + env: + EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} + + steps: + - name: Set up Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + cache-binary: false + + - name: Login to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Re-tag PG17 image as :latest and : + # buildx imagetools preserves multi-arch manifests; `docker tag` would not. + run: | + docker buildx imagetools create \ + -t "${IMAGE}:latest" \ + -t "${IMAGE}:${EQL_VERSION}" \ + "${IMAGE}:17-${EQL_VERSION}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b0c0abec..8bf8fa804 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,31 @@ name: Release JS # # See https://docs.npmjs.com/trusted-publishers#supported-cicd-providers # Enforced by scripts/__tests__/workflow-publish-permissions.test.mjs. +# +# --------------------------------------------------------------------------- +# TWO RELEASE LINES, ONE FILE, AND WHY +# --------------------------------------------------------------------------- +# +# This file publishes the JS packages (changesets), the seven FFI tarballs, and +# — since the EQL release pipeline was ported here — the `@cipherstash/eql` npm +# package plus the SQL release, docs bundle and Postgres image that ship with +# it. They are in one file because npm trusted publishing binds a package to a +# repository AND A WORKFLOW FILENAME: every npm publish in this repository has +# to happen here, whatever else it is part of. See AGENTS.md, "Working on EQL". +# +# `classify` is the first job for the same reason. `workflow_dispatch` was added +# with the EQL port, because the EQL prerelease path is dispatched against a +# release branch — and a dispatch reaches every job in the file, including the +# ones that publish. So the mode is computed once, up front, and the production +# jobs are keyed on it rather than assuming a push to `main`. +# +# WHAT IS INERT TODAY. `@cipherstash/eql` is still published from +# `cipherstash/encrypt-query-language`, so every EQL job below sits behind +# `eql-armed` (`scripts/eql-pipeline-armed.mjs`, derived from +# `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`) as well as behind `gate`, +# which refuses the release outright the moment a frozen package's committed +# version goes missing from npm. Deleting that map entry at the Phase-5 cutover +# arms the whole line; there is no separate flag to remember. permissions: contents: read @@ -22,10 +47,136 @@ on: push: branches: - main + # The EQL prerelease path: dispatched against a release branch whose HEAD is + # an explicit `chore(release): …` marker commit. A dispatch against `main` + # classifies as `production` and behaves exactly like a push — `gate` makes it + # a no-op when there is nothing unpublished. + workflow_dispatch: {} concurrency: ${{ github.workflow }}-${{ github.ref }} jobs: + # WHICH OF THE TWO RELEASE SHAPES THIS RUN IS, decided before anything else + # runs and read by every publishing job below. + # + # production — a push to `main`, or a dispatch against it. Changesets owns + # the versions; the FFI and EQL branches follow whatever it + # published. + # prerelease — a dispatch against any OTHER branch whose HEAD commit + # subject is literally `chore(release):`. Cuts an EQL alpha/rc + # from a batching branch for pre-merge testing, without + # merging to `main`. + # skip — everything else, including a prerelease identity that has + # already been published. + # + # Safety on the prerelease path comes from three guards, not from the branch + # name: the marker commit, a version that must already be prerelease-shaped, + # and the tag short-circuit. Exactly `chore(release):` — a bare `release:` + # prefix must NOT re-trigger the publish pipeline for the pinned identity. + classify: + name: Classify release intent + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + mode: ${{ steps.classify.outputs.mode }} + version: ${{ steps.classify.outputs.version }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + # The classifier reads the HEAD commit subject. + fetch-depth: 0 + # The tag short-circuit below asks the API rather than the remote, so + # this job needs no pushable credential in its checkout. + persist-credentials: false + + - id: classify + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + branch="${GITHUB_REF_NAME}" + subject="$(git log -1 --format=%s)" + mode="skip" + version="" + + if [[ "$branch" == "main" ]]; then + mode="production" + else + case "$subject" in + chore\(release\):*) + mode="prerelease" + ;; + esac + fi + + if [[ "$mode" == "prerelease" ]]; then + # TWO LEVELS DOWN. The EQL subtree root carries no package.json by + # design — upstream this path was `./packages/eql/package.json`, + # because there the repository root WAS the subtree root. Here that + # resolves to a directory with no manifest and the read throws. See + # AGENTS.md, "Working on EQL". + version="$(node -p "require('./packages/eql/packages/eql/package.json').version")" + if [[ "$version" != *-* ]]; then + echo "::error::a prerelease release commit must already pin a prerelease version in packages/eql/packages/eql/package.json" >&2 + exit 1 + fi + # Idempotency short-circuit: if this identity's npm tag already + # exists, the release was cut — a re-push of the marker, or a rerun, + # must not publish it twice. The individual steps are idempotent too; + # this just skips the whole pipeline up front. + # + # `git/matching-refs`, not `git/ref`: the latter answers a + # non-exact match with an ARRAY rather than an object, so `--jq + # .object.sha` yields nothing and the branch below reads "no tag". + # See the same note on `publish-ffi` below. + tag="eql-typescript-v${version}" + at="$(gh api "repos/${REPO}/git/matching-refs/tags/${tag}" \ + --jq ".[] | select(.ref == \"refs/tags/${tag}\") | .object.sha" 2>/dev/null || true)" + if [ -n "$at" ]; then + echo "${tag} already exists; prerelease ${version} was already published — skipping" + mode="skip" + version="" + fi + fi + + echo "classified as ${mode}${version:+ at ${version}}" + { + echo "mode=$mode" + echo "version=$version" + } >> "$GITHUB_OUTPUT" + + # MAY THIS REPOSITORY PUBLISH THE EQL RELEASE LINE AT ALL? + # + # A checkout and one `node` call reading a Map — no registry lookup, so it + # gives the same answer to every EQL job in this run and to `release-plz.yml` + # and `release-postgres-eql-image.yml`, which ask it separately. Read the + # header of scripts/eql-pipeline-armed.mjs for why the switch is DERIVED from + # `FROZEN_PUBLISHERS` rather than being a boolean somebody flips at the + # cutover: a forgotten flag fails silently, in the direction that publishes an + # npm package with no SQL release, no docs and no crate. + eql-armed: + name: Is the EQL release line armed? + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + armed: ${{ steps.armed.outputs.armed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + # No pnpm, no install — the script imports node builtins and + # scripts/release-gate.mjs, which does the same. Same reasoning as `gate`. + - name: Read the publisher switch + id: armed + run: node scripts/eql-pipeline-armed.mjs + # WHAT STILL HAS TO BE PUBLISHED, asked of the registry rather than of the # `.changeset/` directory. "No unconsumed changesets" is also true of an # ordinary docs commit and of the commit right after a release, so gating the @@ -69,16 +220,19 @@ jobs: ffi-artifacts: name: Build FFI artifacts - needs: [gate] - if: needs.gate.outputs.ffi == 'true' + needs: [classify, gate] + # `mode == 'production'` is what `workflow_dispatch` cost. Without it, a + # dispatch against a feature branch whose committed FFI versions happen to + # be unpublished would build and publish them from that branch. + if: needs.classify.outputs.mode == 'production' && needs.gate.outputs.ffi == 'true' uses: ./.github/workflows/_build-ffi-artifacts.yml with: ref: ${{ github.sha }} publish-ffi: name: Publish FFI packages - needs: [gate, ffi-artifacts] - if: needs.gate.outputs.ffi == 'true' + needs: [classify, gate, ffi-artifacts] + if: needs.classify.outputs.mode == 'production' && needs.gate.outputs.ffi == 'true' # GitHub-hosted for the same reason the release job is: npm rejects # provenance from a self-hosted runner with E422. This job only uploads # prebuilt tarballs, so it needs no toolchain beyond node and npm. @@ -216,7 +370,7 @@ jobs: release: name: Release - needs: [gate, publish-ffi] + needs: [classify, gate, publish-ffi] # `always()` because `publish-ffi` is SKIPPED for an ordinary JS release, # and a skipped dependency would otherwise skip this job too. # @@ -229,6 +383,7 @@ jobs: # SUCCEEDED. if: >- always() && + needs.classify.outputs.mode == 'production' && needs.gate.result == 'success' && ( needs.gate.outputs.ffi != 'true' || @@ -242,6 +397,13 @@ jobs: id-token: write # npm OIDC trusted publishing contents: write # changesets commits and pushes the Version Packages branch pull-requests: write # …and opens/updates the PR for it + # What the EQL jobs below need to know, and nothing more. `published` alone + # is not enough: this job publishes every unpublished JS package, and an + # ordinary `@cipherstash/stack` release must not fire an EQL SQL release. + outputs: + eql_published: ${{ steps.eql.outputs.eql_published }} + eql_version: ${{ steps.eql.outputs.eql_version }} + eql_prerelease: ${{ steps.eql.outputs.eql_prerelease }} steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -367,3 +529,389 @@ jobs: # the single go-live switch — set it with: # gh variable set STASH_POSTHOG_KEY --repo cipherstash/stack --body '' STASH_POSTHOG_KEY: ${{ vars.STASH_POSTHOG_KEY }} + + # DID THIS RUN PUBLISH @cipherstash/eql, AND AT WHAT VERSION? + # + # `changesets/action` reports `publishedPackages` as a JSON array of + # `{name, version}`. Reading the version from there rather than from the + # tree is what makes the SQL release, the docs bundle and the image tag + # agree with what actually reached npm — a re-run against a tree whose + # version has already been published finds nothing here and the whole EQL + # branch skips, which is the correct answer. + # + # `publishedPackages` is `[]` when nothing published, and unset when the + # action took its VERSION branch instead (the Version Packages PR), so the + # step is guarded on `published` and defaults to an empty version. + - name: Resolve the published EQL version + id: eql + if: steps.changesets.outputs.published == 'true' + env: + PUBLISHED: ${{ steps.changesets.outputs.publishedPackages }} + run: | + set -euo pipefail + version="$(node -e "const p=JSON.parse(process.env.PUBLISHED);const e=p.find(x=>x.name==='@cipherstash/eql');console.log(e ? e.version : '')")" + if [ -z "$version" ]; then + echo "@cipherstash/eql was not part of this release" + exit 0 + fi + if [[ "$version" == *-* ]]; then + prerelease=true + else + prerelease=false + fi + echo "@cipherstash/eql@${version} published (prerelease=${prerelease})" + { + echo "eql_published=true" + echo "eql_version=${version}" + echo "eql_prerelease=${prerelease}" + } >> "$GITHUB_OUTPUT" + + # ------------------------------------------------------------------------- + # The EQL release line — production + # ------------------------------------------------------------------------- + # + # `@cipherstash/eql` ships as FIVE artefacts at one version: the npm package, + # the `eql-bindings` crate, the SQL bundle, the docs bundle and the + # `postgres-eql` image. Changesets publishes the first (above) and + # `release-plz.yml` publishes the second on the same push; the three below are + # built here, in the run that published, so they cannot drift from it. + # + # ORDERING IS A `needs:` CHAIN, NOT A CONVENIENCE. `eql-docs` attaches to a + # GitHub release that `eql-sql` creates, and `eql-image` dispatches against + # the `eql-` tag that the same job's `target_commitish` produced. + + eql-sql: + name: Build and attach the EQL SQL release + needs: [classify, gate, eql-armed, release] + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' + permissions: + contents: write # creates the eql- tag and release + uses: ./.github/workflows/_build-eql-sql.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.release.outputs.eql_version }} + attach: true + target_commitish: ${{ github.sha }} + prerelease: ${{ needs.release.outputs.eql_prerelease == 'true' }} + + eql-docs: + name: Build and attach the EQL docs bundle + needs: [classify, gate, eql-armed, release, eql-sql] + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' + permissions: + contents: write # attaches to the release eql-sql just created + uses: ./.github/workflows/_build-eql-docs.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.release.outputs.eql_version }} + + eql-image: + name: Dispatch the Postgres + EQL image build + needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] + # PRODUCTION FINALS ONLY: the floating :latest / : image tags must + # not move for a prerelease. An alpha image can still be built on demand + # through release-postgres-eql-image.yml's own workflow_dispatch. + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' && + needs.release.outputs.eql_prerelease == 'false' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write # gh workflow run + steps: + - name: Dispatch release-postgres-eql-image.yml + # Dispatched, not an `on: release` fan-out: a release created by + # GITHUB_TOKEN does not trigger `on: release`, whereas a GITHUB_TOKEN + # workflow_dispatch does. + # + # Against the `eql-` TAG, not the branch. `eql-sql` (a `needs`) + # created that tag at the release commit, so the image is built from the + # exact released source even if `main` has advanced since the publish. + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.release.outputs.eql_version }} + run: | + set -euo pipefail + # This job deliberately has no checkout, so `--repo` is required: + # without it gh tries to infer the repository from a .git directory + # that does not exist. + gh workflow run release-postgres-eql-image.yml \ + --repo "${GITHUB_REPOSITORY}" \ + --ref "eql-${VERSION}" \ + -f eql_version="$VERSION" \ + -f update_floating_tags=true + + # ------------------------------------------------------------------------- + # The EQL release line — prerelease + # ------------------------------------------------------------------------- + # + # Dispatched against a batching branch whose HEAD is a `chore(release):` + # marker, so an alpha/rc can be cut for pre-merge testing without merging to + # `main`. Changesets is not involved: the version is already pinned in the + # package manifest, and `prerelease-eql-npm` publishes it directly. + # + # SQL AND DOCS BUILD FIRST, and the npm publish `needs:` both. The npm tarball + # embeds the SQL assets, so a failure to build them must not arrive after the + # package is on the registry. + + prerelease-eql-sql: + name: Build and attach the prerelease EQL SQL + needs: [classify, gate, eql-armed] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + permissions: + contents: write + uses: ./.github/workflows/_build-eql-sql.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.classify.outputs.version }} + attach: true + target_commitish: ${{ github.sha }} + prerelease: true + + prerelease-eql-docs: + name: Build and attach the prerelease EQL docs + needs: [classify, gate, eql-armed, prerelease-eql-sql] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + permissions: + contents: write + uses: ./.github/workflows/_build-eql-docs.yml + with: + ref: ${{ github.sha }} + tag: eql-${{ needs.classify.outputs.version }} + + prerelease-eql-npm: + name: Publish the prerelease npm package + needs: [classify, gate, eql-armed, prerelease-eql-sql, prerelease-eql-docs] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + # GitHub-hosted for the same reason as `release` and `publish-ffi`: npm + # rejects provenance from a self-hosted runner with E422. + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + id-token: write # npm OIDC trusted publishing + contents: write # creates the eql-typescript-v tag + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + name: Install pnpm + with: + run_install: false + # Never cache the pnpm store in a credential-bearing workflow. + cache: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + # `add_shims_to_path: false` and `env: false` ARE LOAD-BEARING here, for + # exactly the reasons spelled out at length on the `release` job above: + # `packages/eql/mise.toml` pins `node = "22"` under `[tools]` and mise's + # shim directory is PREPENDED to PATH, so the default `true` would shadow + # the Node installed above and hand every later `npm` call mise's bundled + # npm 10.x — which has no OIDC trusted publishing and fails with E404. + # `env: false` keeps that file's `[env]` block (DATABASE_URL and friends, + # pointed at a Postgres this job does not have) out of GITHUB_ENV. + - name: Install mise (release:prepare_bindings_assets runs through it) + uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + cache: false + working_directory: packages/eql + add_shims_to_path: false + env: false + + # AFTER mise-action, deliberately: see the `release` job. + - name: Upgrade npm for OIDC trusted publishing + run: npm install -g npm@^11.5.1 + + # node-pty's install hook falls back to `node-gyp rebuild` when no + # linux-x64 prebuild matches, and pnpm/action-setup v6 no longer ships + # node-gyp on PATH. Upstream's copy of this job had no such step because + # upstream had no node-pty; the install here would die with `sh: 1: + # node-gyp: not found`. Asserted by + # scripts/__tests__/workflow-node-gyp.test.mjs. + - name: Install node-gyp + run: npm install -g node-gyp + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Verify the prerelease marker and version + env: + VERSION: ${{ needs.classify.outputs.version }} + run: | + set -euo pipefail + actual="$(node -p "require('./packages/eql/packages/eql/package.json').version")" + test "$actual" = "$VERSION" || { + echo "::error::package version ${actual} does not match the prerelease identity ${VERSION}" >&2 + exit 1 + } + + - name: Prepare the exact SQL assets + # `--force` is inside the task: `mise run build --version` does not + # treat `--version` as a cache-key input, so on unchanged SQL and Rust + # it would re-serve whatever version the previous build stamped. See + # AGENTS.md, "Working on EQL". + working-directory: packages/eql + env: + VERSION: ${{ needs.classify.outputs.version }} + run: mise run release:prepare_bindings_assets --version "$VERSION" + + # THROUGH TURBO, not `pnpm --filter … build`, which is what upstream ran. + # `@cipherstash/eql#build` declares `dependsOn: ["^build"]`, and a bare + # pnpm filter builds only that package — so its workspace dependencies are + # whatever an earlier step happened to leave behind. Asserted by + # scripts/__tests__/workflow-turbo-build-deps.test.mjs. + - name: Build the package + run: pnpm exec turbo run build --filter @cipherstash/eql + + - name: Publish the package + # TWO LEVELS DOWN — the npm package, not the subtree root. Upstream this + # was `packages/eql`, where the repository root was the subtree root. + working-directory: packages/eql/packages/eql + env: + VERSION: ${{ needs.classify.outputs.version }} + run: | + set -euo pipefail + if [ -n "$(npm view "@cipherstash/eql@${VERSION}" version 2>/dev/null)" ]; then + echo "@cipherstash/eql@${VERSION} is already published; skipping publish" + else + node scripts/npm-publish.mjs + fi + + - name: Tag the TypeScript release + env: + VERSION: ${{ needs.classify.outputs.version }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # Created through the REST API rather than with git: the checkout + # carries `persist-credentials: false` (deliberate hardening) and a + # git-level extraheader workaround proved unreliable upstream ("could + # not read Username"). `classify` reads this same tag to decide + # whether the identity was already released. + tag="eql-typescript-v${VERSION}" + at="$(gh api "repos/${REPO}/git/matching-refs/tags/${tag}" \ + --jq ".[] | select(.ref == \"refs/tags/${tag}\") | .object.sha" 2>/dev/null || true)" + if [ -n "$at" ]; then + echo "tag ${tag} already exists at ${at} — skipping" + else + gh api -X POST "repos/${REPO}/git/refs" \ + -f ref="refs/tags/${tag}" -f sha="$GITHUB_SHA" >/dev/null + echo "created ${tag} at ${GITHUB_SHA}" + fi + + prerelease-eql-crate: + name: Dispatch the prerelease crate publish + needs: [classify, gate, eql-armed, prerelease-eql-sql, prerelease-eql-docs] + if: >- + needs.classify.outputs.mode == 'prerelease' && + needs.eql-armed.outputs.armed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write # gh workflow run + contents: write # create/update the release/eql- branch ref + steps: + - name: Dispatch release-plz.yml at the release commit + # release-plz refuses a detached HEAD ("cannot determine current + # branch"), so dispatching against the tag does not work. To publish + # from the EXACT commit the SQL and npm artefacts shipped from — even if + # the branch has advanced since the marker push — pin a + # `release/eql-` branch at this run's commit and dispatch + # against that. It is a stable pointer per identity: force-updated on a + # retry of the same identity, never reused across identities. + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.classify.outputs.version }} + REPO: ${{ github.repository }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + branch="release/eql-${VERSION}" + if ! gh api -X POST "repos/${REPO}/git/refs" \ + -f ref="refs/heads/${branch}" -f sha="$SHA" >/dev/null 2>&1; then + gh api -X PATCH "repos/${REPO}/git/refs/heads/${branch}" \ + -f sha="$SHA" -F force=true >/dev/null + fi + # `--repo` is required: this job has no checkout, and without it gh + # tries to resolve the workflow through a local git repo. + gh workflow run release-plz.yml --repo "$REPO" --ref "$branch" + + # ------------------------------------------------------------------------- + + # WHAT THIS RUN ACTUALLY DID, in one place. + # + # Worth a job rather than leaving it to the run page because both EQL paths + # can legitimately do NOTHING: a dispatch whose prerelease identity is already + # tagged classifies as `skip`, and every job below `classify` is then skipped + # for a reason the run page renders identically to "a dependency failed". This + # says which it was. + eql-summary: + name: EQL release summary + needs: + - classify + - eql-armed + - eql-sql + - eql-docs + - eql-image + - prerelease-eql-sql + - prerelease-eql-docs + - prerelease-eql-npm + - prerelease-eql-crate + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Emit the run summary + env: + MODE: ${{ needs.classify.outputs.mode }} + VERSION: ${{ needs.classify.outputs.version }} + ARMED: ${{ needs.eql-armed.outputs.armed }} + EQL_SQL: ${{ needs.eql-sql.result }} + EQL_DOCS: ${{ needs.eql-docs.result }} + EQL_IMAGE: ${{ needs.eql-image.result }} + PRE_SQL: ${{ needs.prerelease-eql-sql.result }} + PRE_DOCS: ${{ needs.prerelease-eql-docs.result }} + PRE_NPM: ${{ needs.prerelease-eql-npm.result }} + PRE_CRATE: ${{ needs.prerelease-eql-crate.result }} + run: | + set -euo pipefail + { + echo "## EQL release" + echo "" + echo "- mode: \`${MODE}\`" + echo "- pipeline armed: \`${ARMED}\`" + echo "- classified version: \`${VERSION:-n/a}\`" + echo "" + echo "| job | result |" + echo "| --- | --- |" + echo "| eql-sql | ${EQL_SQL} |" + echo "| eql-docs | ${EQL_DOCS} |" + echo "| eql-image | ${EQL_IMAGE} |" + echo "| prerelease-eql-sql | ${PRE_SQL} |" + echo "| prerelease-eql-docs | ${PRE_DOCS} |" + echo "| prerelease-eql-npm | ${PRE_NPM} |" + echo "| prerelease-eql-crate | ${PRE_CRATE} |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index 4f5bc9482..fc774a344 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -289,23 +289,64 @@ monorepo, which is where the silent failures are. because two task scripts use `git rev-parse --show-toplevel`, which after the import returns the **monorepo** root — and one of them, `tasks/test/doc-anchors.sh`, fails silently when that is wrong. -- **`packages/eql/.github/` is a dead deposit.** GitHub reads workflows from the - repo root and nowhere else, so the eleven files there run on nothing. Seven - are workflows — four publish something (`release.yml`, `release-plz.yml`, - `release-postgres-eql-image.yml`, `rebuild-docs.yml`) and three exist only to - serve them (two `workflow_call` reusables and `lint-release.yml`) — alongside - a `workflows/README.md`, the release-notes config, and two - repository-settings files. `scripts/__tests__/eql-suite-ci.test.mjs` holds - them as a **shrinking allowlist** (`UNPORTED_DEPOSIT`), asserted by equality so - it fails in both directions: porting a workflow means deleting it from the - deposit *and* from the list in the same commit, and dropping a new file in - there without listing it fails too. The same test asserts that the three SQLx - suite tasks are invoked by name from a root workflow, that every - `dorny/paths-filter` path is scoped to `packages/eql/`, and that every mise - task shelling out to cargo is either reachable from a root workflow or - exempted with a written reason. It is the guard against the failure this - absorption keeps rediscovering: a check that arrives as a file and executes on - no event reads exactly like a check that passes. +- **`packages/eql/.github/` is gone, and a test fails if it returns.** It was a + dead deposit: GitHub reads workflows from the repo root and nowhere else, so + the eleven files that arrived with the subtree ran on nothing. + `scripts/__tests__/eql-suite-ci.test.mjs` held them as a shrinking allowlist + until the release port emptied it, and now asserts the directory does not + exist and that nothing is tracked under it. **Never put a workflow under a + package** — it executes on no event, which reads exactly like a check that + passes. That same test also asserts that the three SQLx suite tasks are + invoked by name from a root workflow, that every `dorny/paths-filter` path is + scoped to `packages/eql/`, and that every mise task shelling out to cargo is + either reachable from a root workflow or exempted with a written reason. + That scan now follows a task's `tasks/*.sh` delegations **transitively** + (cycle-guarded, and it throws rather than truncating past + `MAX_SCRIPT_DEPTH`), and it reads the `run:` bodies of composite actions a + workflow reaches through `uses: ./…`. Both used to stop at one hop, and both + failure directions were live: a cargo helper reached only at depth 2 dropped + out of `CARGO_TASKS` entirely — no orphan reported, no exemption demanded, + and the job running it stopped counting as a Rust job for the cache check — + while a mise task invoked from a composite action read as run by nobody, + whose natural repair is an exemption claiming CI does not run it. +- **The EQL release pipeline is built and INERT, and the switch that arms it is + derived rather than flipped.** Five artefacts ship at one version — the npm + package, the `eql-bindings` crate, the SQL bundle, the docs bundle and the + `postgres-eql` image — and the workflows that produce them all live at the + repo root now: + + | file | what it does | + |---|---| + | `release.yml` | `classify` + the EQL production and prerelease jobs. It is one file with the JS and FFI releases because npm trusted publishing binds a package to a repository **and a workflow filename** | + | `_build-eql-sql.yml`, `_build-eql-docs.yml` | `workflow_call` reusables, called from both EQL paths so there is one build path per artefact | + | `release-plz.yml` | the crate. **Its filename cannot change** — crates.io Trusted Publishing binds to it | + | `release-postgres-eql-image.yml` | the GHCR image, dispatched by `release.yml` on production finals | + | `rebuild-docs.yml`, `lint-release.yml` | merged into the root files of the same name | + + Every one of those sits behind `scripts/eql-pipeline-armed.mjs`, which reads + `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs` and answers "may this + repository publish EQL at all?". **Deleting the `@cipherstash/eql` entry at + the Phase-5 cutover is what arms the pipeline** — there is deliberately no + second flag, because a forgotten one fails silently in the direction that + publishes an npm package with no SQL release, no docs and no crate. The gate + itself is the other half: it exits non-zero the moment a frozen package's + committed version is missing from npm, which skips `release` and everything + downstream of it. Read `node scripts/release-gate.mjs` for what it is doing + right now rather than inferring it from here. + Two things the cutover also needs, neither of which a workflow can assert + ahead of time: a `GPG_PRIVATE_KEY` secret in this repository (release-plz + signs its commit and tag), and write access from this repository to the + `ghcr.io/cipherstash/postgres-eql` package, which is currently linked to + `cipherstash/encrypt-query-language`. +- **A release workflow may not restore a cache, including a Rust one, and the + two rules that say so now know about each other.** + `scripts/lint-no-workflow-caching.mjs` forbids a GitHub Actions cache restore + anywhere an artefact is published; `eql-suite-ci.test.mjs` requires a + `Swatinem/rust-cache` step on every job that compiles Rust. Four EQL release + jobs do both, so they pay a cold cargo compile — and the exemption is derived + from the linter's own target list (read out of its success output, since the + script lints on import) rather than copied, so the two cannot disagree about + which jobs those are. That scan now follows a task's `tasks/*.sh` delegations **transitively** (cycle-guarded, and it throws rather than truncating past `MAX_SCRIPT_DEPTH`), and it reads the `run:` bodies of composite actions a @@ -445,7 +486,9 @@ monorepo, which is where the silent failures are. `@cipherstash/eql@3.0.5` records that repository and `.github/workflows/release.yml` — and the package's own `repository` / `bugs` fields still point there, as do the `eql-bindings` crate's. Repointing all of - it is the Phase-5 cutover, together with the nine parked workflows above. + it is the Phase-5 cutover. **The pipeline it repoints at already exists** — + see "The EQL release pipeline is built and INERT" above; what is left is the + registry-side move plus deleting the map entry that keeps it inert. This was protect-ffi's situation until its own cutover, with one difference: the guard is not a changeset lint but `scripts/release-gate.mjs`, and it is the stronger of the two. Its `FROZEN_PUBLISHERS` map lists every package that lives here but is diff --git a/SECURITY.md b/SECURITY.md index 08d349c34..13a7833a5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -150,7 +150,18 @@ CI. See `skills/stash-supply-chain-security/SKILL.md` for the full guide. The `release.yml` workflow publishes packages to npm using OIDC trusted publishing (`id-token: write`). There is no long-lived `NPM_TOKEN` — the workflow deliberately avoids one, and setting one would bypass trusted -publishing. +publishing. `release-plz.yml` publishes the `eql-bindings` crate to crates.io +over the same token exchange, and likewise carries no `CARGO_REGISTRY_TOKEN`. +Both bind to a *workflow filename* at the registry, so renaming either file +silently invalidates its publisher configuration. + +`scripts/__tests__/workflow-publish-permissions.test.mjs` holds the shape those +two files must keep: `id-token: write` is granted per job and never at workflow +level (where it would be a default inherited by every job in a file the registry +already trusts), and a job in such a file that does not publish may not hold a +writable scope. Both lists are equalities, so a new publisher — or a new job +that can write to the repository the publishers build from — has to be argued +for in the same diff. [GitHub Actions cache poisoning is a known attack][1] against credential-bearing workflows. The mechanism is: @@ -163,8 +174,17 @@ workflows. The mechanism is: We mitigate this by: -- Explicitly disabling all caching in `release.yml` +- Explicitly disabling all caching in every workflow that publishes an artefact + — `release.yml`, the reusables it calls (`_build-ffi-artifacts.yml`, + `_build-eql-sql.yml`, `_build-eql-docs.yml`), `release-plz.yml` and + `release-postgres-eql-image.yml` - Automated checks for disabled caching on high-risk workflows + (`scripts/lint-no-workflow-caching.mjs`). It works from an ALLOWLIST of + audited actions rather than a denylist of cache actions, so an action it has + never seen is a finding by default — the cases that matter most are + `setup-` actions that cache by default, with no `cache:` input and no + telling name. The cost is real and accepted: four EQL release jobs compile + Rust with no `Swatinem/rust-cache` restore. [1]: https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/ diff --git a/docs/plans/2026-08-13-eql-monorepo-absorption.md b/docs/plans/2026-08-13-eql-monorepo-absorption.md index 19e30c5d1..c626e201b 100644 --- a/docs/plans/2026-08-13-eql-monorepo-absorption.md +++ b/docs/plans/2026-08-13-eql-monorepo-absorption.md @@ -306,23 +306,23 @@ The protect-ffi lesson applies verbatim: **a check that arrives as a file and th - [x] Satisfy this repo's credentialed-job guards for the two `CS_*`-bearing jobs — `BINDING_EXEMPT_JOBS` entries with the crate-not-binding reason, a `require-cs-secrets` step before the credentialed work, and the `vars`/`secrets` split this repo uses. See the finding above; `scripts/__tests__/ffi-binding-step-order.test.mjs` fails the build until all three are done. - [x] Fork-PR-skip the credentialed jobs, matching the other `integration-*.yml` workflows. Already true upstream — but **two of this repo's guards had to be extended rather than exempted**, which the plan did not anticipate. `workflow-dispatch-job-conditions.test.mjs` compared whole conditions and required exactly one spelling; EQL's two jobs `&&` the guard with a relevance gate, so the single-spelling rule now applies to the *clause*, compared verbatim, with another conjunct permitted beside it. Its synthetic contexts gained `merge_group` and a permissive `needs` (without which the compound conditions read as skipped on every event), and the verdict table is now derived from the workflow's declared triggers — `test-eql.yml` has no `push:`, and asserting about a run that cannot happen is not a check. Its evaluator also had to model `always()` for the `ci-required` aggregator: total on every event, therefore modellable; `success()` and argument-taking calls still throw. - [x] Port `bench-eql.yml` and `macro-expand-eql.yml`. `bench` needed three things beyond paths: it holds `CS_*`, so it takes the `vars`/`secrets` split and a `require-cs-secrets` pre-flight (worth more on a scheduled 60-minute job than on a PR — nobody watches a nightly start); and its three actions were on floating major tags, now SHA-pinned to the same commits `test-eql.yml` uses. It also joins `EXPECTED_ASYMMETRIES` in the paths-filter parity guard: `push` only, no second list to drift from. -- [ ] **Moved to Phase 5** — `lint-release.yml` and the two reusable workflows, plus the `cache: false` additions and the `lint-no-workflow-caching.mjs` `TARGETS`/`AUDITED_ACTIONS` work. All of it is about the release machinery: EQL's `lint-release.yml` points actionlint at four release workflows that do not exist at the root yet, and `_build-sql.yml` / `_build-docs.yml` are `workflow_call`-only, reached from EQL's `release.yml` alone. Porting them here would land a gate aimed at absent files. The one genuinely Phase-4 piece is its `shellcheck` + `prepare-bindings-assets.test.sh` half, which covers the asset builder the root `version` script already calls — carry that into the root `lint-release.yml` rather than leaving it with the rest. +- [x] **Moved to Phase 5, and done there** — `lint-release.yml` and the two reusable workflows, plus the `cache: false` additions and the `lint-no-workflow-caching.mjs` `TARGETS`/`AUDITED_ACTIONS` work. All of it is about the release machinery: EQL's `lint-release.yml` points actionlint at four release workflows that do not exist at the root yet, and `_build-sql.yml` / `_build-docs.yml` are `workflow_call`-only, reached from EQL's `release.yml` alone. Porting them here would land a gate aimed at absent files. The one genuinely Phase-4 piece is its `shellcheck` + `prepare-bindings-assets.test.sh` half, which covers the asset builder the root `version` script already calls — carry that into the root `lint-release.yml` rather than leaving it with the rest. - [x] Split cargo off the default test path exactly as protect-ffi did. **Half of this was already true and the other half had nowhere to attach.** EQL has no npm-script layer over its cargo work — the checks are mise tasks invoked straight from workflows — so `@cipherstash/eql`'s `test` is `vitest run` and there is no `test:cargo` to be reachable from. The property therefore lands one level up, in `scripts/__tests__/eql-suite-ci.test.mjs`: every mise task that invokes cargo must be reached by a root workflow, directly or through `depends`. 19 tasks qualify (15 if you read only the task block — the rest hide behind `run = "bash tasks/test/*.sh"`, which is the class most worth checking, so the scan follows that hop and its floor is set to fail if it stops). Five are unreached and each is named with its reason: a watcher, two `:regen` write-halves whose read-halves run in CI, an unsharded local variant, and a unit-test task `test:crates` subsumes. - [x] Add the EQL Cargo workspace to Dependabot. **The existing lockfile check would not have caught this**: `supply-chain.e2e.test.ts` asserts coverage per *ecosystem*, and `cargo` was already covered by the protect-ffi entry, so `packages/eql/Cargo.lock` read as monitored while nothing proposed an update for it. Dependabot's cargo `directory:` is one workspace root, not a glob. The `ignore` list carries the same four CipherStash crates, and here the argument is sharper than analogy: `tests/sqlx/Cargo.toml` pins `cipherstash-client = "=0.42.0"`, the same exact pin at the same version as protect-ffi, so a PR moving one workspace and not the other reintroduces exactly the skew this absorption removes. - [x] `.github/actionlint.yaml` must declare the Blacksmith runner labels the imported workflows use — one label, `blacksmith-16vcpu-ubuntu-2204`, used by every job in the suite. Verified by running actionlint over the ported files: identical shellcheck findings before and after, and nothing structural. -- [ ] **Moved to Phase 5 — an ordering defect, found by executing this phase.** Four of the ten workflows *are* the release machinery, and Phase 5 ports them; deleting the directory here would mean reconstructing a publish pipeline from git history at the one moment nobody wants to be doing that. What lands now instead is the shrinking allowlist in `eql-suite-ci.test.mjs`: every file still in the deposit, named with the reason it has not been ported. It is an equality, so it fails in both directions — a file that comes back fails, and so does an entry left behind after its file is gone, which is what forces the last removal to be `rm -r` plus a straight `existsSync(...) === false`. +- [x] **Moved to Phase 5, and done there — an ordering defect, found by executing this phase.** Four of the ten workflows *are* the release machinery, and Phase 5 ports them; deleting the directory here would mean reconstructing a publish pipeline from git history at the one moment nobody wants to be doing that. What lands now instead is the shrinking allowlist in `eql-suite-ci.test.mjs`: every file still in the deposit, named with the reason it has not been ported. It is an equality, so it fails in both directions — a file that comes back fails, and so does an entry left behind after its file is gone, which is what forces the last removal to be `rm -r` plus a straight `existsSync(...) === false`. - [x] **Write the test that asserts a root workflow still runs the SQLx suite** — `scripts/__tests__/eql-suite-ci.test.mjs`, in the shape of `packages/protect-ffi/src/integrationSuiteCi.test.ts` and reusing its `executablePart` trick, so a `paths:` entry or a comment naming a task cannot satisfy it. Mutation-checked six ways across this phase: unprefixing a paths-filter entry, deleting the partition step, demoting it to a comment, pointing rust-cache back at the monorepo root, dropping the e2e pre-flight, and respelling the fork clause with its disjuncts swapped — each fails the intended guard and only that one. ## Phase 5 — release cutover Five surfaces, each flipped independently, each inert until its own cutover. -- [ ] **Carried over from Phase 4, deliberately.** Port `lint-release.yml` and the two reusable workflows (`_build-sql.yml`, `_build-docs.yml`) alongside the release workflows they gate and are called from — a lint aimed at four absent files is worse than no lint. Add `cache: false` to the three `jdx/mise-action` steps that omit it (`_build-sql.yml`, `_build-docs.yml`, `release-postgres-eql-image.yml`), and add those two plus `release-plz.yml` to `scripts/lint-no-workflow-caching.mjs`'s `TARGETS`, with their remote actions in `AUDITED_ACTIONS` — or the gate fails on the addition itself. -- [ ] **Then delete `packages/eql/.github/` entirely**, and convert the shrinking allowlist in `scripts/__tests__/eql-suite-ci.test.mjs` to a straight `existsSync(...) === false`, alongside the existing protect-ffi assertion in `lintWiring.test.ts`. The allowlist is an equality, so it will already be failing by the time the last file goes — that is what makes this step forced rather than remembered. +- [x] **Carried over from Phase 4, deliberately.** Port `lint-release.yml` and the two reusable workflows (`_build-sql.yml`, `_build-docs.yml`) alongside the release workflows they gate and are called from — a lint aimed at four absent files is worse than no lint. Add `cache: false` to the three `jdx/mise-action` steps that omit it (`_build-sql.yml`, `_build-docs.yml`, `release-postgres-eql-image.yml`), and add those two plus `release-plz.yml` to `scripts/lint-no-workflow-caching.mjs`'s `TARGETS`, with their remote actions in `AUDITED_ACTIONS` — or the gate fails on the addition itself. **Two corrections from doing it.** The three files did not omit `cache:`; they set it to `true`, one hit each, so the fix was to flip the value rather than add a key (adding one produces a duplicate YAML key). And the reusables were renamed on the way in — `_build-eql-sql.yml`, `_build-eql-docs.yml` — because "SQL" and "docs" name nothing on their own in a monorepo and nothing binds to either filename; `release-plz.yml` is the one that could not be renamed. `release-plz.yml` also lost its mise step entirely: no step in that job runs a mise task, and the publishing-job rule (`add_shims_to_path: false`) would have left mise's toolchain off PATH anyway. +- [x] **Then delete `packages/eql/.github/` entirely**, and convert the shrinking allowlist in `scripts/__tests__/eql-suite-ci.test.mjs` to a straight `existsSync(...) === false`, alongside the existing protect-ffi assertion in `lintWiring.test.ts`. The allowlist is an equality, so it will already be failing by the time the last file goes — that is what makes this step forced rather than remembered. - [x] ~~Add `scripts/lint-no-eql-changeset.mjs` in the shape of the FFI guard~~ — **superseded, and do not restore the parked changeset.** The guard landed instead as `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`, which is strictly stronger: it exits non-zero on any frozen package whose committed version is missing from npm *and* on any runtime `workspace:` range only that package could satisfy, a pair of conditions no changeset-side guard can see — which is why the hand-applied 3.0.5 bump needed this guard rather than that one. (Run `node scripts/release-gate.mjs` for what it blocks at any given moment; that answer comes from the registry and is not recorded here.) It runs at PR time via `tests.yml` and at release time via `release.yml`. `.changeset/rename-ste-vec-contains.md.deferred` was deleted deliberately in `9b1c44d9` — the 3.0.5 bump was entered by hand in the CHANGELOG, so re-parking that `major` changeset would make the cutover bump the package a second time (3.0.5 → 4.0.0) for a rename already released. **Phase-5 action: delete the `@cipherstash/eql` entry from `FROZEN_PUBLISHERS`, nothing more.** - [ ] Repoint all EQL manifests at `cipherstash/stack`: `packages/eql/packages/eql/package.json` — `repository.url`, `repository.directory` → `packages/eql/packages/eql`, **and `bugs.url`** (line 13, still `encrypt-query-language/issues`, which would send users to an archived repo) — plus `crates/eql-bindings/Cargo.toml` (`repository`, `homepage`) and `release-plz.toml`'s prose. npm and crates.io both reject a publish whose repository field does not match; `bugs.url` is not enforced, which is why it is the one that survives a cutover unnoticed. -- [ ] Port `release.yml`'s EQL half into this repo's `release.yml` — the existing gate-then-publish structure already models "one version authority plus N idempotent publishers", which is the shape it was derived from. Preserve the `classify` prerelease path and the `eql-typescript-v` idempotency short-circuit. -- [ ] Port `release-plz.yml` unchanged in filename — crates.io trusted publishing binds to it. +- [x] Port `release.yml`'s EQL half into this repo's `release.yml` — the existing gate-then-publish structure already models "one version authority plus N idempotent publishers", which is the shape it was derived from. Preserve the `classify` prerelease path and the `eql-typescript-v` idempotency short-circuit. **What it cost, all of it found by this repo's guards rather than by review.** `workflow_dispatch` on `release.yml` (the prerelease path is dispatched) means every job in the file is reachable by hand, so `classify` gates the FFI and changesets jobs too. Three guards had to grow: `workflow-publish-permissions.test.mjs` needed `REPO_WRITE_JOBS` beside `OIDC_JOBS`, because six new jobs need a writable scope and must NOT be able to publish; `workflow-dispatch-job-conditions.test.mjs` needed a reasoned skip list, because the production and prerelease halves are mutually exclusive by construction and no single synthetic context can run both; and `eql-suite-ci.test.mjs`'s rust-cache requirement needed a publish-path exemption derived from the caching linter's own targets. Two real defects came out of the same pass: the prerelease job needed `Install node-gyp` before `pnpm install` (node-pty, which upstream did not have) and `pnpm exec turbo run build` rather than `pnpm --filter … build` (`^build` dependencies, which upstream did not have either). +- [x] Port `release-plz.yml` unchanged in filename — crates.io trusted publishing binds to it. It also needed `manifest_path: packages/eql/Cargo.toml` and `config: packages/eql/release-plz.toml`, which upstream did not: a composite action's steps do not inherit the caller's `defaults.run.working-directory`, so a `working-directory:` on the step would have done nothing and release-plz would have read the monorepo root, found no workspace naming `eql-bindings`, and exited 0 having published nothing. - [ ] Port `release-postgres-eql-image.yml` and `rebuild-docs.yml`. - [ ] **Repoint trusted publishing**, only after a versioned dry run: npm for `@cipherstash/eql` (workflow `release.yml`, explicitly selecting `npm publish` under Allowed actions), and crates.io for `eql-bindings` (workflow `release-plz.yml`). - [ ] Move the `DOCS_WEBHOOK_URL` secret and the GHCR package ownership. @@ -359,11 +359,11 @@ Five surfaces, each flipped independently, each inert until its own cutover. - [x] `cargo build -p protect-ffi` succeeds against the path-dep `eql-bindings` — host (4m 51s) and `wasm32-unknown-unknown` (1m 18s), plus `test:cargo` at 310 passed / 0 failed and `cargo fmt --check` clean - [x] The credentialed protect-ffi integration suite passes unchanged against the path dep. **Split out of the line above deliberately**: they were one box, and one box that is half-runnable is a box that gets ticked. Closed by runs [`31759054388`](https://github.com/cipherstash/stack/actions/runs/31759054388) and [`32316091388`](https://github.com/cipherstash/stack/actions/runs/32316091388) on PR #885 — the credentialed, Dockerised `integration-protect-ffi.yml`, green at both heads that carried the flip. Scope and limits are on the Phase-3 box of the same name; do not restate the verdict without them - [x] No manifest names `eql-bindings` or `@cipherstash/eql` by registry version, except `packages/protect-ffi/integration-tests` under a named exemption with a written reason — `scripts/lint-no-eql-registry-pins.mjs`, run from `tests.yml`'s `lint` job, mutation-checked four ways -- [ ] `packages/eql/.github/` is deleted and a test fails if it returns — **Phase 5**, once the release workflows are ported. Until then `eql-suite-ci.test.mjs` holds the deposit as an equality against the list of files still waiting. +- [x] `packages/eql/.github/` is deleted and a test fails if it returns. Three of the eleven files were dropped rather than ported, each with the reason recorded where the allowlist used to be: the actionlint config and the issue template are duplicates the root already carries, and `workflows/README.md` documented a merge-queue model this repository does not have. - [x] The SQLx matrix runs from a root workflow, fork-PR-skipped, credentialed, PG 14–17 in the merge queue - [x] `scripts/__tests__/ffi-binding-step-order.test.mjs` passes with the EQL jobs present — exempted from the binding build with a reason, **not** exempted from `require-cs-secrets` -- [ ] No `jdx/mise-action` step in any release workflow omits `cache:`, and the caching lint's target list covers `release-plz.yml` and the image publisher -- [x] `actionlint` clean over every ported workflow — verified for `test-eql.yml`, `bench-eql.yml` and `macro-expand-eql.yml`. "Clean" means no structural, expression or runner-label findings; the shellcheck style/info diagnostics are upstream's and are byte-for-byte the same set before and after the port, which is the comparison that shows the port introduced nothing +- [x] No `jdx/mise-action` step in any release workflow restores a cache, and the caching lint's target list covers all four ported release workflows. `AUDITED_ACTIONS` gained seven entries; two of the three Docker ones turned out to have a GitHub-Actions-cache input defaulting to **true** (`cache-image`, `cache-binary`), and `docker/build-push-action` — which has no such toggle — is held by a new `forbiddenInputs` rule on `cache-from` / `cache-to`, since `type=gha` names that cache directly +- [x] `actionlint` clean over every ported workflow — verified for `test-eql.yml`, `bench-eql.yml` and `macro-expand-eql.yml`, and now for the four release workflows too, which `lint-release.yml` lints on every PR that touches them rather than leaving it to a local run. "Clean" means no structural, expression or runner-label findings; the shellcheck style/info diagnostics are upstream's and are byte-for-byte the same set before and after the port, which is the comparison that shows the port introduced nothing - [ ] `release.yml` passes `version: pnpm run version` to `changesets/action`, and a dry run shows `Cargo.toml` and the SQL assets moving to the computed version — the hook is dead without it - [ ] All EQL manifests read `cipherstash/stack`, including `repository.directory` and `bugs.url` - [ ] `pnpm run code:check` error-free across the merged tree diff --git a/packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml b/packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml deleted file mode 100644 index 49af6b6fb..000000000 --- a/packages/eql/.github/ISSUE_TEMPLATE/docs-feedback.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Docs feedback -description: Feedback to help make our docs more effective. -title: "[Docs]: " -labels: ["docs", "triage"] -assignees: - - kateandrews -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to give us feedback! - - type: input - id: contact - attributes: - label: Contact details - description: How can we get in touch with you if we need more info? - placeholder: you@example.com - validations: - required: false - - type: textarea - id: what-problem - attributes: - label: What problem were you trying to solve? - placeholder: Tell us what you were looking for in our docs. - validations: - required: true - - type: textarea - id: additional-info - attributes: - label: Anything else you'd like us to know? - placeholder: Let us know if you've got any additional feedback. - validations: - required: false - - type: checkboxes - id: terms - attributes: - label: Code of conduct - description: By submitting this issue, you agree to follow our [Code of conduct](https://github.com/cipherstash/encrypt-query-language/blob/main/CODE_OF_CONDUCT.md). - options: - - label: I agree to follow this project's Code of conduct - required: true diff --git a/packages/eql/.github/actionlint.yaml b/packages/eql/.github/actionlint.yaml deleted file mode 100644 index b6a2e7da2..000000000 --- a/packages/eql/.github/actionlint.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# actionlint does not know repo-specific / provider-specific runner labels. -self-hosted-runner: - labels: - - blacksmith-16vcpu-ubuntu-2204 diff --git a/packages/eql/.github/release.yml b/packages/eql/.github/release.yml deleted file mode 100644 index ec475e66b..000000000 --- a/packages/eql/.github/release.yml +++ /dev/null @@ -1,5 +0,0 @@ -changelog: - categories: - - title: Changes since the last release - labels: - - "*" diff --git a/packages/eql/.github/workflows/README.md b/packages/eql/.github/workflows/README.md deleted file mode 100644 index cbe72a1c5..000000000 --- a/packages/eql/.github/workflows/README.md +++ /dev/null @@ -1,228 +0,0 @@ -# CI workflows - -This directory holds every GitHub Actions workflow for EQL. This README is the -authoritative **inventory** (what workflows exist and when they fire) and -**coverage map** (which job runs which checks, and where each test suite -actually runs). - -- [Workflow inventory](#workflow-inventory) -- [`test-eql.yml` — the merge gate](#test-eqlyml--the-merge-gate) -- [Coverage map: job → task → what it checks](#coverage-map-job--task--what-it-checks) -- [Where each test suite runs](#where-each-test-suite-runs) -- [Known gaps](#known-gaps) - ---- - -## Workflow inventory - -| Workflow | Triggers | What it does | Gates merge? | -|---|---|---|---| -| **test-eql.yml** | `pull_request`, `merge_group`, `workflow_dispatch` | Full test/lint/validate matrix; the one required check | **Yes** — `ci-required` | -| **release-postgres-eql-image.yml** | `workflow_dispatch` (dispatched by `release.yml` on production finals) | Build & push the Postgres+EQL Docker image to GHCR | No | -| **release.yml** | `push: main`, `push: eql_v3`, `workflow_dispatch` | Unified release entrypoint: production on `main`, prerelease on `eql_v3` when the commit is an explicit `chore(release): ...` marker | No | -| **release-plz.yml** | `push: main`, `workflow_dispatch` | Publish the `eql-bindings` crate to crates.io (Trusted Publishing) + open the release PR | No | -| **bench-eql.yml** | `push: main` (paths), `schedule` 02:00 UTC daily, `workflow_dispatch` | `test:bench` (bench cargo feature). **Never runs on PRs** | No | -| **macro-expand-eql.yml** | `schedule` 03:00 UTC daily, `workflow_dispatch` | Regenerate the integer `cargo expand` matrix snapshot; needs pinned nightly | **No — explicitly non-blocking** | -| **rebuild-docs.yml** | `push: tags` | Fire a docs-site rebuild webhook | N/A | - -Only **test-eql.yml** gates merges. Bench regressions and stale `cargo expand` -snapshots surface on the nightly schedule, not on the PR that caused them. - -### Two release tag families - -There are two independent release flows keyed on distinct git tags: - -- **`eql-`** (e.g. `eql-3.0.0`) — the EQL **SQL surface** release, cut - by `release.yml` (production on `main`, prerelease on `eql_v3`), which builds - and attaches the SQL + docs in-run. On production finals `release.yml` also - dispatches `release-postgres-eql-image.yml` (the Postgres+EQL Docker image). -- **`eql-bindings-v`** (e.g. `eql-bindings-v0.1.0`) — the **`eql-bindings` - Rust crate** release, cut automatically by `release-plz.yml` when its release - PR merges. Publishes to crates.io only. - -The three SQL-surface workflows explicitly **exclude** `eql-bindings-*` tags -(`!startsWith(...'eql-bindings')` guards; a `!eql-bindings-*` filter in -`rebuild-docs.yml`) so a crate release never triggers an SQL build, Docker -image, or docs-site rebuild. `eql-bindings` is tested on every PR by the -`rust-crates` job (`mise run test:crates` plus a `cargo publish --dry-run` -packaging gate); every other workspace crate is `publish = false`. - ---- - -## `test-eql.yml` — the merge gate - -Fast PR feedback + a thorough pre-merge gate, using a merge queue and a single -aggregated required check. - -### Two run shapes - -`setup` derives the matrix from the event: - -| Event | Trigger | Matrix | Purpose | -|---|---|---|---| -| `pull_request` | push to a PR | PG17 × 4 shards | fast developer feedback | -| `merge_group` | "Merge when ready" → queued | PG14–17 × 2 shards | full pre-merge gate on the real merged state | -| `workflow_dispatch` | manual run | PG17 × 4 shards (PR shape) | ad-hoc | - -The PR run is feedback only. The merge-queue run is the gate. - -> **No `push` trigger.** Under a required merge queue, push-to-main validation is -> redundant — the queue already validated the exact merge commit, and branch -> protection blocks direct pushes. - -### Relevance skip applies to PRs only - -Each heavy job runs when: -`merge_group || workflow_dispatch || (pull_request && relevant == 'true')`. - -So the `changes` relevance filter only gates the **`pull_request`** event — a -docs-only PR skips the heavy jobs on its PR run. On `merge_group` (and -`workflow_dispatch`) every job runs **unconditionally**: a queued PR always pays -the full gate regardless of which files it touched. - -The relevance filter (`changes` job) marks a PR relevant when any of these -changed: `.github/workflows/test-eql.yml`, `src/**`, `sql/**`, `tests/**`, -`tasks/**`, `crates/**`, `Cargo.toml`, `Cargo.lock`, `mise.toml`. Note `docs/**` -is **not** in the filter — see [Known gaps](#known-gaps). - -### How the queue works - -1. Click **Merge when ready** — the PR is queued, not merged. -2. GitHub builds a temporary branch = `main` + this PR (+ any PRs ahead in the - queue) and fires `merge_group`, so CI tests the **post-merge state**. -3. The full PG14–17 × 2 matrix (plus the single-run jobs) runs and feeds - `ci-required`. -4. `ci-required` green → PR is **merged**. Red → PR is **removed from the - queue**; `main` is untouched. - -This catches semantic conflicts — two PRs that each pass alone but break -together — which PR-only checks never test. - -### The `ci-required` aggregator - -Per-event matrices make leaf job names unstable (`Shard PG17 1/4` on a PR vs. -`Shard PG14 1/2` in the queue), so leaf names can't be named as required checks. -Instead, one aggregator job (id and display name `ci-required`) `needs:` every -job, runs with `if: always()`, and passes only if each needed result is -`success` **or** `skipped`. Mark **only `ci-required`** as the required status -check. - -- `if: always()` — runs even when dependencies fail/skip, so the check always - reports (a never-reported required check leaves the queue stuck *Pending*). -- `skipped` counts as pass — a docs-only PR skips the heavy jobs on its PR run - but must still report Success so the PR stays eligible to queue. - ---- - -## Coverage map: job → task → what it checks - -All jobs run on `blacksmith-16vcpu-ubuntu-2204`. "PG set" follows the event -(PG17 on PR / dispatch, PG14–17 in the queue). - -| Job | mise task(s) | Checks | DB | `CS_*` creds | -|---|---|---|---|---| -| **changes** | — | compute relevance | no | no | -| **setup** | — | compute PG × shard matrix | no | no | -| **build-archive** | `test:sqlx:archive` | Build EQL, run prep, **generate fixtures**, compile every `tests/sqlx` binary (**default features**) into a nextest archive; upload archive + `release/*.sql` | yes (PG17) | **yes (sole holder)** | -| **test** (sharded) | `test:sqlx:partition` | Run the archived sqlx binaries (default features), hash-partitioned across shards | yes (per PG) | no (replays archive) | -| **e2e** | `test:sqlx:e2e` | The `proptest-e2e` fresh-encryption suites (`e2e_oracle` **and** `float_special`) — PG17 only, version-independent | yes (PG17) | **yes** | -| **known-failures** | `test:known-failures:parser` + `test:known-failures` | The gate's strict `ISSUE_` parser is sound; every `known_failure` marker names a real, OPEN issue and is referenced by a test (DB-free, needs `gh`) | no | no | -| **doc-anchors** | `test:doc-anchors` | Every intra-document markdown anchor link resolves to a real heading (DB-free; not relevance-gated — its inputs are the docs) | no | no | -| **validate** (per PG) | `docs:validate:documented-sql` + `test:clean_install_v3` | DB-backed SQL doc-syntax check; clean-DB `eql_v3` install smoke | yes | no | -| **docs-static** | `docs:validate:source` + `test:docs_v3_grep` + `test:public_identifiers` | SQL doxygen coverage + required tags; user docs teach only v3; tracked public files contain no private issue identifiers or tracker links | no | no | -| **schema** | `test:schema` | v2.2 / v2.3 payload JSON-schema validation | no | no | -| **rust-crates** | `test:crates` + `types:check` | `cargo fmt --check`, clippy + `cargo test` for `eql-domains` / `eql-codegen` / `eql-tests-macros` / `eql-bindings`; verify TS bindings + JSON schemas are fresh | no | no | -| **codegen** | `codegen:parity` | Regenerate encrypted-domain SQL in place + `git diff` drift gate (committed `src/v3/scalars/` matches the generator) | no | no | -| **self-contained-v3** | `test:self_contained_v3`, `test:installer_complete`, `test:symbol_order_v3`, `test:build_ordering_helpers` | `eql_v3` surface has no `eql_v2` dependency; installer contains every ordered file; singleton symbols defined before use (overloads are resolved exactly by the `clean-install` job's `test:clean_install_v3`) | no | no | -| **matrix-coverage** | `test:matrix:inventory` (+`:jsonb_entry`, `:v3-jsonb`) + `test:matrix:catalog-coverage` | Scalar-matrix test-name snapshots are not silently dropped; catalog surface is covered | no | no | -| **splinter** | `test:splinter` | Supabase/Splinter lints over the installed EQL | yes (PG17) | no | -| **ci-required** | — | aggregator: every needed job is `success`/`skipped` | no | no | - ---- - -## Where each test suite runs - -The `eql_v3` property-test suites (see -`tests/sqlx/tests/encrypted_domain/property/README.md`) land in three different -CI jobs: - -| Suite | Job | Trigger coverage | DB | `CS_*` | Notes | -|---|---|---|---|---|---| -| **catalog** (`eql-domains` `proptest_invariants`) | **rust-crates** (`cargo test -p eql-domains`; proptest is a dev-dep) | relevant PR + queue | no | no | pure-Rust catalog invariants; shrinking enabled | -| **fixture** (function-double oracles, extractor identity, `match_smoke`, `edge_cases`) | **test** shards (default features) | relevant PR (PG17×4) + queue (PG14–17×2) | yes | no | oracle over the **committed** real-ciphertext fixtures | -| **e2e** (`e2e_oracle`, `#[cfg(feature = "proptest-e2e")]`) | **e2e** job (`test:sqlx:e2e`) | relevant PR (PG17) + queue (PG17) | yes | yes | oracle over **fresh** ZeroKMS encryption; PG-version-independent, so one PG17 run | -| **float specials** (`float_special`, `#[cfg(feature = "proptest-e2e")]`) | **e2e** job (`test:sqlx:e2e`) | relevant PR (PG17) + queue (PG17) | yes | yes | NaN / ±0.0 / ±Inf encoder tripwires over **fresh** encryption | - -The wider sqlx suite (everything under `tests/sqlx/tests/`) runs in the **test** -shards, which replay the default-feature archive — so any -`#[cfg(feature = …)]`-gated test that isn't in the default feature set does not -run there. `proptest-e2e` is the one such gate, and it has its own **e2e** job -(it can't reuse the credential-free archive: it both compiles with a non-default -feature and needs `CS_*` at run time). - -> **The e2e job filters by test name, so every `proptest-e2e` module must be -> named there explicitly.** `test:sqlx:e2e` runs `cargo test --features -> proptest-e2e e2e_oracle` *and* `… float_special`. A bare unfiltered run would -> duplicate the whole sharded fixture suite; but for a long time the job filtered -> on `e2e_oracle` alone, and `float_special` — gated behind the same feature, so -> never compiled by the default-feature shards either — ran in **no** CI job at -> all from the day it was written. Adding a new `proptest-e2e`-gated module means -> adding a line to `test:sqlx:e2e`. Nothing globs it for you. - ---- - -## Known gaps - -1. **bench + macro-expand are nightly / non-blocking** — a bench regression or a - stale `cargo expand` snapshot surfaces on the daily schedule, not on the PR - that introduced it. Accepted trade-off. - -2. **`docs/**` markdown is not content-validated.** The `docs-static` job checks - the SQL `--!` doxygen comments under `src/**`, not the prose/links in `docs/**` - itself. A markdown-only PR leaves `relevant` false, so `docs-static` is skipped - along with the other heavy jobs — and that loses no coverage, because the job's - inputs (`src/**` `.sql`/`.template`, the `crates/**` codegen build, the - `tasks/docs/**` scripts) are all in the `relevant` filter, so a PR that doesn't - trip `relevant` cannot change its outcome. Linting the markdown the PR actually - changed (prose/links) is a separate, unfilled capability. - -### Recently closed - -- *The e2e (fresh-encryption) suite never ran in CI.* Now covered by the **e2e** - job (`test:sqlx:e2e`), PG17, on relevant PRs + the queue. -- *`docs-static` ran unconditionally on every PR.* It is now relevance-gated like - every other heavy job. Because its inputs are a strict subset of the `relevant` - filter, gating it both makes the workflow consistent (one uniform `if:`) and - drops a redundant codegen build on markdown-only PRs without losing any - coverage. A narrower bespoke `src/**`-only filter was rejected: it would risk a - silent false-green (`ci-required` counts `skipped` as pass) by skipping on a - real input change in `crates/**` or `tasks/docs/**`. - ---- - -## Operator setup (one-time, GitHub UI) - -Settings → Branches → rule for `main`: - -1. **Require merge queue.** -2. **Require status checks to pass** → add **`ci-required` only** (not the - per-shard leaf names). - -Then verify (see `docs/plans/2026-06-09-ci-pr-feedback-sharding-rollout.md`): - -- **Queue a relevant PR** → `merge_group` runs the full gate (8 `Shard …` jobs + - 4 `Validate …` jobs + `build-archive`, `e2e`, `docs-static`, `schema`, - `rust-crates`, `codegen`, `self-contained-v3`, `matrix-coverage`, `splinter`) → - `ci-required` green → PR merges. -- **Open a docs-only PR** → on its `pull_request` run every relevance-gated heavy - job skips (`docs-static` included); `ci-required` reports **Success** (not stuck - *Pending*) because it counts `skipped` as pass, so the PR can be queued. - -## References - -- Merge queue: -- `merge_group` event: -- Required status checks: -- `needs` / `always()` / `join()`: -- Path filtering (`dorny/paths-filter`): -- nextest archive + partitioning: · diff --git a/packages/eql/.github/workflows/_build-docs.yml b/packages/eql/.github/workflows/_build-docs.yml deleted file mode 100644 index 8a30f98ad..000000000 --- a/packages/eql/.github/workflows/_build-docs.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: "Build docs (reusable)" - -# Reusable docs build+attach. Called inline by release.yml (production and -# prerelease paths). The target release already exists: _build-sql.yml creates -# it in the same run before this docs job attaches to it. - -on: - workflow_call: - inputs: - ref: - description: "Git ref/SHA to build docs from. Empty -> default checkout (github.sha)." - required: false - type: string - default: "" - tag: - description: "Full release tag. Empty -> build only, no attach." - required: false - type: string - default: "" - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - -defaults: - run: - shell: bash {0} - -permissions: - contents: write - -jobs: - publish-docs: - runs-on: blacksmith-16vcpu-ubuntu-2204 - name: Build and Publish Documentation - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ inputs.ref }} - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - cache: true - - - name: Install Doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - - name: Generate documentation - env: - TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - mise run docs:generate - mise run docs:generate:markdown -- "${TAG}" - mise run docs:generate:json -- "${TAG}" - - - name: Package documentation - env: - TAG: ${{ inputs.tag }} - run: | - mise run docs:package "${TAG}" - - - name: Upload documentation artifacts - uses: actions/upload-artifact@v4 - with: - name: eql-docs - path: | - release/eql-docs-*.zip - release/eql-docs-*.tar.gz - - - name: Publish documentation to release - if: ${{ inputs.tag != '' }} - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag }} - files: | - release/eql-docs-*.zip - release/eql-docs-*.tar.gz diff --git a/packages/eql/.github/workflows/_build-sql.yml b/packages/eql/.github/workflows/_build-sql.yml deleted file mode 100644 index bd5977978..000000000 --- a/packages/eql/.github/workflows/_build-sql.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: "Build SQL (reusable)" - -# Reusable SQL build+attach. Called inline by release.yml (production and -# prerelease paths) so SQL releases have one build path. - -on: - workflow_call: - inputs: - ref: - description: "Git ref/SHA to build from. Empty -> default checkout (github.sha)." - required: false - type: string - default: "" - tag: - description: "Full release tag (e.g. eql-3.0.0-alpha.2). Empty -> DEV build, no attach." - required: false - type: string - default: "" - attach: - description: "Attach the built .sql artefacts to a GitHub Release." - required: false - type: boolean - default: false - target_commitish: - description: "Non-empty -> create a prerelease at this commit; empty -> attach to the existing release named by tag." - required: false - type: string - default: "" - prerelease: - description: "Mark the created release as a prerelease (create path only)." - required: false - type: boolean - default: false - secrets: - MULTITUDES_ACCESS_TOKEN: - required: false - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - -defaults: - run: - shell: bash {0} - -permissions: - contents: write - -jobs: - build: - runs-on: blacksmith-16vcpu-ubuntu-2204 - name: Build EQL - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ inputs.ref }} - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - cache: true - - - name: Build EQL release - # Strip `eql-` so eql_v3.version() reports bare semver. Empty TAG - # intentionally falls through build.sh's ${usage_version:-DEV} default. - env: - TAG: ${{ inputs.tag }} - run: | - mise run build --version "${TAG#eql-}" - - - name: Upload EQL artifacts - uses: actions/upload-artifact@v4 - with: - name: eql-release - path: | - release/cipherstash-encrypt.sql - release/cipherstash-encrypt-uninstall.sql - - - name: Attach artefacts to existing release - if: ${{ inputs.attach && inputs.target_commitish == '' }} - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag }} - files: | - release/cipherstash-encrypt.sql - release/cipherstash-encrypt-uninstall.sql - - - name: Create prerelease at commit - if: ${{ inputs.attach && inputs.target_commitish != '' }} - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag }} - target_commitish: ${{ inputs.target_commitish }} - prerelease: ${{ inputs.prerelease }} - name: ${{ inputs.tag }} - body: "Preview (prerelease) of the standalone eql_v3 surface. See [Unreleased] in CHANGELOG.md." - files: | - release/cipherstash-encrypt.sql - release/cipherstash-encrypt-uninstall.sql - - - name: Notify Multitudes - # Production deploy tracking: fire only when release.yml creates a - # non-prerelease SQL release (attach + create path + not a prerelease). - # release.yml runs on `push`, so the old `event_name == 'release'` gate - # never matched under the unified workflow. - # continue-on-error: a deploy-tracking ping is not part of the release - # itself. If Multitudes is down / rate-limits / the token is empty, the - # curl failing must not fail build-sql and skip downstream build-docs / - # build-image after npm + SQL have already published. - if: ${{ inputs.attach && !inputs.prerelease && inputs.target_commitish != '' }} - continue-on-error: true - run: | - curl --request POST \ - --fail-with-body \ - --url "https://api.developer.multitudes.co/deployments" \ - --header "Content-Type: application/json" \ - --header "Authorization: ${{ secrets.MULTITUDES_ACCESS_TOKEN }}" \ - --data '{"commitSha": "${{ github.sha }}", "environmentName":"production"}' diff --git a/packages/eql/.github/workflows/lint-release.yml b/packages/eql/.github/workflows/lint-release.yml deleted file mode 100644 index b63c21091..000000000 --- a/packages/eql/.github/workflows/lint-release.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: "Lint release tooling" - -# Durable gate for the release machinery. It catches workflow syntax, wrapper -# shell issues, and supply-chain cache regressions before a real release. - -on: - pull_request: - paths: - - .github/workflows/_build-sql.yml - - .github/workflows/_build-docs.yml - - .github/workflows/release-plz.yml - - .github/workflows/release.yml - - .github/workflows/lint-release.yml - - .github/actionlint.yaml - - package.json - - scripts/*.mjs - - tasks/release/*.sh - workflow_dispatch: {} - -permissions: - contents: read - -defaults: - run: - shell: bash - -jobs: - lint: - name: actionlint + shellcheck + unit test - runs-on: blacksmith-16vcpu-ubuntu-2204 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Install actionlint - run: | - set -euo pipefail - bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.7/scripts/download-actionlint.bash) 1.7.7 - echo "$PWD" >> "$GITHUB_PATH" - - - name: actionlint (release workflows) - run: | - set -euo pipefail - actionlint \ - .github/workflows/_build-sql.yml \ - .github/workflows/_build-docs.yml \ - .github/workflows/release-plz.yml \ - .github/workflows/release.yml \ - .github/workflows/lint-release.yml - - - name: shellcheck (wrappers + helpers) - run: | - set -euo pipefail - shellcheck \ - tasks/release/prepare-bindings-assets.sh \ - tasks/release/prepare-bindings-assets.test.sh - - - name: prepare-bindings-assets validation unit test - run: bash tasks/release/prepare-bindings-assets.test.sh - - - uses: pnpm/action-setup@v6.0.8 - name: Install pnpm - with: - run_install: false - cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Install JS dependencies - run: pnpm install --frozen-lockfile - - - name: release workflow supply-chain cache guard - run: | - pnpm run lint:workflow-cache - pnpm run test:scripts diff --git a/packages/eql/.github/workflows/rebuild-docs.yml b/packages/eql/.github/workflows/rebuild-docs.yml deleted file mode 100644 index 3353299d9..000000000 --- a/packages/eql/.github/workflows/rebuild-docs.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Rebuild Docs - -on: - push: - tags: - - 'eql-*' - # Exclude the eql-bindings crate tags (eql-bindings-v*) cut by release-plz - # — a Rust-crate release must not fire the docs-site rebuild webhook. - - '!eql-bindings-*' - -jobs: - trigger-docs-rebuild: - name: Trigger Docs Rebuild - runs-on: blacksmith-16vcpu-ubuntu-2204 - steps: - - name: Send webhook - env: - WEBHOOK_URL: ${{ secrets.DOCS_WEBHOOK_URL }} - run: | - curl -X POST "$WEBHOOK_URL" diff --git a/packages/eql/.github/workflows/release-plz.yml b/packages/eql/.github/workflows/release-plz.yml deleted file mode 100644 index a24248619..000000000 --- a/packages/eql/.github/workflows/release-plz.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: "Release eql-bindings (crates.io)" - -# Publishes the `eql-bindings` crate to crates.io via release-plz (crates.io -# Trusted Publishing via OIDC — no long-lived token; GPG-signed release -# commits/tags; blacksmith runner + mise toolchain). -# -# Publish-only: crate versioning is owned by CHANGESETS, not release-plz. The -# changesets "Version Packages" PR bumps crates/eql-bindings/Cargo.toml to the -# lockstep version V (scripts/sync-lockstep-versions.mjs) alongside the npm -# package. On push to main, `release` publishes the committed Cargo.toml version -# V to crates.io (no-op when V is already published) and tags eql-bindings-vV. -# There is deliberately NO release-plz `release-pr` job — changesets opens the -# version PR, so a release-plz PR would fight it. -# -# One-time crates.io setup (Trusted Publishing): on the eql-bindings crate's -# Settings -> Trusted Publishing, add a GitHub publisher with -# Repository: cipherstash/encrypt-query-language Workflow: release-plz.yml -# For a brand-new crate name, register the publisher before the first publish. -# -# Secrets: GPG_PRIVATE_KEY (release-commit/tag signing key). crates.io auth is -# OIDC (id-token: write), so NO CARGO_REGISTRY_TOKEN. The GitHub PR/tag ops use -# the default GITHUB_TOKEN; enable "Allow GitHub Actions to create and approve -# pull requests" in repo settings so the release PR can be opened. - -permissions: - pull-requests: write - contents: write - id-token: write # crates.io Trusted Publishing (OIDC) - -on: - push: - branches: - - main - workflow_dispatch: {} - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - -# No `defaults.run.shell` override: GitHub's default `bash` shell runs with -# `-eo pipefail` — the right default for a publish workflow. - -# Static group: push-to-main and workflow_dispatch share it so two release-plz -# invocations never race a tag/publish. Never cancel — a cancelled release could -# leave a half-published state. -concurrency: - group: release-plz - cancel-in-progress: false - -jobs: - release: - name: "Release" - runs-on: blacksmith-16vcpu-ubuntu-2204 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Sign release-plz's commits and tags with the shared release key. - - name: Import GPG key - uses: crazy-max/ghaction-import-gpg@v7 - with: - gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - git_user_signingkey: true - git_commit_gpgsign: true - git_tag_gpgsign: true - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - # Publish workflows must not restore caches (lint-no-workflow-caching - # covers mise-action): a poisoned toolchain cache would run inside the - # job that holds crates.io OIDC publishing power. - cache: false - - # The crate bundles its SQL via include_str! on COMMITTED files, and - # release-plz publishes the committed tree verbatim. Refuse to publish a - # crate whose bundled SQL wasn't prepared for this exact version — e.g. a - # hand-pinned Cargo.toml without running `pnpm run version` would - # otherwise ship the DEV placeholder as its "exact SQL". Enforced only - # when this run would actually publish (the committed version is not on - # crates.io yet): routine no-op runs on main between releases carry the - # DEV placeholder legitimately. A crates.io API failure fails towards - # ENFORCING the guard, never towards skipping it. - - name: Verify bundled SQL matches the crate version - run: | - set -euo pipefail - cargo_version="$(grep -m1 '^version = ' crates/eql-bindings/Cargo.toml | cut -d'"' -f2)" - # crates.io rejects curl's default User-Agent with a 403. Without an - # explicit UA the request always fails, `|| true` swallows it, and the - # no-op short-circuit below is unreachable. - published="$(curl -fsSL --retry 3 -H "User-Agent: cipherstash-eql-release (https://github.com/cipherstash/encrypt-query-language)" "https://crates.io/api/v1/crates/eql-bindings/versions" | jq -r '.versions[].num' | grep -Fx "$cargo_version" || true)" - if [[ -n "$published" ]]; then - echo "eql-bindings@${cargo_version} is already on crates.io; this run is a publish no-op — skipping the bundled-SQL guard" - exit 0 - fi - manifest_version="$(jq -r '.eqlVersion' crates/eql-bindings/sql/release-manifest.json)" - if [[ "$manifest_version" != "$cargo_version" ]]; then - echo "::error::crates/eql-bindings/sql/release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml version ('$cargo_version') — the bundled SQL was not prepared for this release. Run 'pnpm run version' (or 'mise run release:prepare_bindings_assets --version $cargo_version') and commit the result." >&2 - exit 1 - fi - if ! grep -q "eql_v3" crates/eql-bindings/sql/cipherstash-encrypt.sql; then - echo "::error::crates/eql-bindings/sql/cipherstash-encrypt.sql looks like the DEV placeholder — refusing to publish it as release SQL." >&2 - exit 1 - fi - - - name: Run release-plz release - uses: release-plz/action@v0.5 - with: - command: release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/packages/eql/.github/workflows/release-postgres-eql-image.yml b/packages/eql/.github/workflows/release-postgres-eql-image.yml deleted file mode 100644 index a311fd703..000000000 --- a/packages/eql/.github/workflows/release-postgres-eql-image.yml +++ /dev/null @@ -1,169 +0,0 @@ -name: "Release Postgres + EQL image" - -on: - # Driven by release.yml on production finals (it dispatches this workflow with - # update_floating_tags=true), plus manual runs. Not `on: release`: a - # GITHUB_TOKEN-created release does not trigger `on: release` fan-out, whereas - # a GITHUB_TOKEN workflow_dispatch does — so release.yml dispatches instead. - workflow_dispatch: - inputs: - eql_version: - description: "EQL version to embed (e.g. 2.1.8). Used as the image tag and SQL build version." - required: true - type: string - update_floating_tags: - description: "Also update :latest, :, and : floating tags. Off by default for test runs." - required: false - type: boolean - default: false - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - MISE_VERBOSE: "1" - IMAGE: ghcr.io/cipherstash/postgres-eql - -defaults: - run: - shell: bash {0} - -permissions: - contents: read - packages: write - -jobs: - build-sql: - runs-on: blacksmith-16vcpu-ubuntu-2204 - name: Build EQL SQL - timeout-minutes: 5 - outputs: - # Used for image tags, e.g. "2.1.8" -> ghcr.io/.../postgres-eql:17-2.1.8 - eql_version: ${{ steps.ver.outputs.eql_version }} - # Passed to `mise run build --version`. Bare semver (no `eql-` prefix), - # matching the SQL release build (_build-sql.yml), so eql_v3.version() in - # the image is byte-identical to the SQL release. - build_version: ${{ steps.ver.outputs.build_version }} - update_floating_tags: ${{ steps.ver.outputs.update_floating_tags }} - - steps: - - uses: actions/checkout@v4 - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - cache: true - - - name: Compute EQL version and tag policy - id: ver - # eql_version is passed in as bare semver (release.yml sends - # needs.release.outputs.version, e.g. "3.0.0"), used both as the image - # tag and, verbatim, as the `mise run build --version` value so - # eql_v3.version() in the image is byte-identical to the SQL release. - env: - INPUT_VERSION: ${{ inputs.eql_version }} - INPUT_FLOATING: ${{ inputs.update_floating_tags }} - run: | - { - echo "eql_version=${INPUT_VERSION}" - echo "build_version=${INPUT_VERSION}" - echo "update_floating_tags=${INPUT_FLOATING}" - } >> "$GITHUB_OUTPUT" - - - name: Build EQL release SQL - run: mise run build --version ${{ steps.ver.outputs.build_version }} - - - name: Upload EQL SQL artifact - uses: actions/upload-artifact@v4 - with: - name: eql-sql - path: release/cipherstash-encrypt.sql - - build-images: - runs-on: blacksmith-16vcpu-ubuntu-2204 - needs: build-sql - name: Build postgres-eql image (PG ${{ matrix.pg_version }}) - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - pg_version: ["14", "15", "16", "17"] - env: - EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} - UPDATE_FLOATING: ${{ needs.build-sql.outputs.update_floating_tags }} - - steps: - - uses: actions/checkout@v4 - - - name: Download EQL SQL artifact - uses: actions/download-artifact@v4 - with: - name: eql-sql - path: docker/ - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Compute image tags - id: tags - # Always push the immutable :- tag. - # Only push the floating : tag on release, or on dispatch with opt-in. - run: | - tags="${IMAGE}:${{ matrix.pg_version }}-${EQL_VERSION}" - if [[ "${UPDATE_FLOATING}" == "true" ]]; then - tags+=$'\n'"${IMAGE}:${{ matrix.pg_version }}" - fi - { - echo 'tags<> "$GITHUB_OUTPUT" - - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: docker - file: docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - build-args: | - PG_VERSION=${{ matrix.pg_version }} - EQL_VERSION=${{ env.EQL_VERSION }} - tags: ${{ steps.tags.outputs.tags }} - - promote-latest: - runs-on: blacksmith-16vcpu-ubuntu-2204 - needs: [build-sql, build-images] - name: Promote PG17 image to :latest and : - if: ${{ needs.build-sql.outputs.update_floating_tags == 'true' }} - timeout-minutes: 5 - env: - EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} - - steps: - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Re-tag PG17 image as :latest and : - # buildx imagetools preserves multi-arch manifests; `docker tag` would not. - run: | - docker buildx imagetools create \ - -t "${IMAGE}:latest" \ - -t "${IMAGE}:${EQL_VERSION}" \ - "${IMAGE}:17-${EQL_VERSION}" diff --git a/packages/eql/.github/workflows/release.yml b/packages/eql/.github/workflows/release.yml deleted file mode 100644 index 9067cf853..000000000 --- a/packages/eql/.github/workflows/release.yml +++ /dev/null @@ -1,426 +0,0 @@ -name: "Release" - -# Single release entry point for the EQL release line. -# -# - `main` uses Changesets to version and publish the production release. -# - Any non-`main` branch can cut a prerelease from an explicit conventional -# release commit, triggered via workflow_dispatch against that branch (e.g. -# `gh workflow run release.yml --ref eql-3.0.1`). The release commit itself is -# the marker: `chore(release): ...`. This lets a batching branch ship an -# alpha/rc for pre-merge testing without merging to `main`. -# -# Prereleases publish the npm package directly from this workflow and dispatch -# the Rust crate publish through `release-plz.yml` so crates.io Trusted -# Publishing still sees the correct entry-point workflow. The SQL GitHub -# release and docs are built in the same workflow run. -# -# npm Trusted Publishing must be configured for this workflow filename: -# `release.yml`. - -permissions: - id-token: write # npm OIDC trusted publishing - contents: write # changesets commits the Version PR / tags; SQL release attach - pull-requests: write # changesets opens the Version PR - -on: - push: - branches: - - main - workflow_dispatch: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -# No `defaults.run.shell` override: GitHub's default `bash` shell runs with -# `-eo pipefail`, which is exactly what a publish workflow wants. (An explicit -# `bash {0}` would silently DISABLE errexit/pipefail for any step that forgets -# its own `set -euo pipefail`.) - -jobs: - classify: - name: Classify release intent - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - mode: ${{ steps.classify.outputs.mode }} - version: ${{ steps.classify.outputs.version }} - prerelease: ${{ steps.classify.outputs.prerelease }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - id: classify - run: | - set -euo pipefail - branch="${GITHUB_REF_NAME}" - subject="$(git log -1 --format=%s)" - mode="skip" - version="" - prerelease="false" - - if [[ "$branch" == "main" ]]; then - mode="production" - else - # Any non-main branch cuts a prerelease from an explicit - # conventional release commit (dispatched via workflow_dispatch). - # Safety comes from the three guards below — the marker commit, the - # prerelease-shaped version, and the tag-idempotency short-circuit — - # not from the branch name. Exactly `chore(release):`; a bare - # `release:` prefix must NOT re-trigger the publish pipeline for the - # currently pinned identity. - case "$subject" in - chore\(release\):*) - mode="prerelease" - ;; - esac - fi - - if [[ "$mode" == "prerelease" ]]; then - version="$(node -p "require('./packages/eql/package.json').version")" - if [[ "$version" != *-* ]]; then - echo "::error::prerelease release commits must already pin a prerelease version in packages/eql/package.json" >&2 - exit 1 - fi - # Idempotency short-circuit: if this identity's npm tag already - # exists, the release was already cut — a re-push of the marker (or - # a rerun) must not re-publish. Individual steps are also - # idempotent; this just skips the whole pipeline up front. - if git ls-remote --exit-code --tags origin "refs/tags/eql-typescript-v${version}" >/dev/null 2>&1; then - echo "eql-typescript-v${version} already exists; prerelease ${version} was already published — skipping" - mode="skip" - version="" - else - prerelease="true" - fi - fi - - { - echo "mode=$mode" - echo "version=$version" - echo "prerelease=$prerelease" - } >> "$GITHUB_OUTPUT" - - release: - name: Version or publish (changesets) - needs: classify - if: ${{ needs.classify.outputs.mode == 'production' }} - # GitHub-hosted (not Blacksmith): npm provenance from OIDC trusted - # publishing is only accepted from github-hosted runners (self-hosted -> E422). - runs-on: ubuntu-latest - timeout-minutes: 20 - outputs: - published: ${{ steps.changesets.outputs.published }} - version: ${{ steps.ver.outputs.version }} - prerelease: ${{ steps.ver.outputs.prerelease }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: pnpm/action-setup@v6.0.8 - name: Install pnpm - with: - run_install: false - cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - # Publish workflows must not restore caches (lint-no-workflow-caching - # covers mise-action too): a poisoned toolchain cache would run inside - # the job that holds npm OIDC publishing power. - cache: false - - # AFTER mise-action, deliberately: mise's node (from [tools]) is first on - # PATH and its bundled npm 10.x has no OIDC trusted publishing — upgrading - # before mise-action upgrades an npm that never runs (the exact cause of - # the alpha.3 ENEEDAUTH failures). - - name: Upgrade npm for OIDC trusted publishing - run: npm install -g npm@11.5.1 - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - # `version:` runs `changeset version && sync-lockstep-versions.mjs` (bumps - # package.json + Cargo.toml + builds/bundles SQL to V) for the Version PR; - # `publish:` runs `pnpm run build && changeset publish` to publish npm. - - name: Version or publish - id: changesets - uses: changesets/action@v1.8.0 - with: - version: pnpm run version - publish: pnpm run release - commitMode: "github-api" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Resolve published version - id: ver - if: steps.changesets.outputs.published == 'true' - env: - PUBLISHED: ${{ steps.changesets.outputs.publishedPackages }} - run: | - set -euo pipefail - version="$(node -e "const p=JSON.parse(process.env.PUBLISHED);const e=p.find(x=>x.name==='@cipherstash/eql');if(!e){process.exit(1)}console.log(e.version)")" - echo "version=${version}" >> "$GITHUB_OUTPUT" - if [[ "$version" == *-* ]]; then - echo "prerelease=true" >> "$GITHUB_OUTPUT" - else - echo "prerelease=false" >> "$GITHUB_OUTPUT" - fi - - build-sql: - name: Build + attach SQL release - needs: release - if: ${{ needs.release.outputs.published == 'true' }} - permissions: - contents: write - secrets: - MULTITUDES_ACCESS_TOKEN: ${{ secrets.MULTITUDES_ACCESS_TOKEN }} - uses: ./.github/workflows/_build-sql.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.release.outputs.version }} - attach: true - target_commitish: ${{ github.sha }} - prerelease: ${{ needs.release.outputs.prerelease == 'true' }} - - build-docs: - name: Build + attach docs - needs: [release, build-sql] - if: ${{ needs.release.outputs.published == 'true' }} - permissions: - contents: write - uses: ./.github/workflows/_build-docs.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.release.outputs.version }} - - build-image: - name: Dispatch Postgres + EQL image build - needs: [release, build-sql, build-docs] - # Production finals only: the floating :latest / : image tags must - # not move for a prerelease. Alpha images can still be built on demand via - # release-postgres-eql-image.yml's workflow_dispatch. Dispatched (not an - # `on: release` fan-out) because a GITHUB_TOKEN-created release does not - # trigger `on: release`, whereas a GITHUB_TOKEN workflow_dispatch does. - if: ${{ needs.release.outputs.published == 'true' && needs.release.outputs.prerelease == 'false' }} - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - actions: write - steps: - - name: Dispatch release-postgres-eql-image.yml - # Dispatch against the eql- tag, not the branch: build-sql - # (a `needs`) already created that tag at the release commit via - # _build-sql.yml's target_commitish, and the image workflow file exists - # at that commit. Pinning to the tag builds the image from the exact - # released source even if the branch has advanced since publish. - env: - GH_TOKEN: ${{ github.token }} - VERSION: ${{ needs.release.outputs.version }} - run: | - set -euo pipefail - # This job deliberately has no checkout. Pass --repo explicitly so - # gh does not try to infer the repository from a nonexistent .git - # directory. - gh workflow run release-postgres-eql-image.yml \ - --repo "${GITHUB_REPOSITORY}" \ - --ref "eql-${VERSION}" \ - -f eql_version="$VERSION" \ - -f update_floating_tags=true - - prerelease-build-sql: - name: Build + attach prerelease SQL - needs: classify - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - permissions: - contents: write - secrets: - MULTITUDES_ACCESS_TOKEN: ${{ secrets.MULTITUDES_ACCESS_TOKEN }} - uses: ./.github/workflows/_build-sql.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.classify.outputs.version }} - attach: true - target_commitish: ${{ github.sha }} - prerelease: true - - prerelease-build-docs: - name: Build + attach prerelease docs - needs: [classify, prerelease-build-sql] - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - permissions: - contents: write - uses: ./.github/workflows/_build-docs.yml - with: - ref: ${{ github.sha }} - tag: eql-${{ needs.classify.outputs.version }} - - prerelease-publish-npm: - name: Publish prerelease npm package - needs: [classify, prerelease-build-sql, prerelease-build-docs] - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: pnpm/action-setup@v6.0.8 - name: Install pnpm - with: - run_install: false - cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: jdx/mise-action@v3 - with: - version: 2026.4.0 - install: true - # Publish workflows must not restore caches (lint-no-workflow-caching - # covers mise-action too): a poisoned toolchain cache would run inside - # the job that holds npm OIDC publishing power. - cache: false - - # AFTER mise-action, deliberately: mise's node (from [tools]) is first on - # PATH and its bundled npm 10.x has no OIDC trusted publishing — upgrading - # before mise-action upgrades an npm that never runs (the exact cause of - # the alpha.3 ENEEDAUTH failures). - - name: Upgrade npm for OIDC trusted publishing - run: npm install -g npm@11.5.1 - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Verify prerelease marker and version - env: - VERSION: ${{ needs.classify.outputs.version }} - run: | - set -euo pipefail - actual="$(node -p "require('./packages/eql/package.json').version")" - test "$actual" = "$VERSION" || { - echo "package version ${actual} does not match prerelease identity ${VERSION}" >&2 - exit 1 - } - - - name: Prepare exact SQL assets - env: - VERSION: ${{ needs.classify.outputs.version }} - run: mise run release:prepare_bindings_assets --version "$VERSION" - - - name: Build package - run: pnpm --filter @cipherstash/eql build - - - name: Publish package - working-directory: packages/eql - env: - VERSION: ${{ needs.classify.outputs.version }} - run: | - set -euo pipefail - if [ -n "$(npm view "@cipherstash/eql@${VERSION}" version 2>/dev/null)" ]; then - echo "@cipherstash/eql@${VERSION} is already published; skipping publish" - else - node scripts/npm-publish.mjs - fi - - - name: Tag TypeScript release - env: - VERSION: ${{ needs.classify.outputs.version }} - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - # Create the tag via the REST API with GH_TOKEN — the checkout has - # persist-credentials: false (deliberate hardening) and a git-level - # extraheader hack proved unreliable ("could not read Username"). - tag="eql-typescript-v${VERSION}" - if gh api "repos/${REPO}/git/ref/tags/${tag}" >/dev/null 2>&1; then - echo "tag ${tag} already exists on the remote; skipping" - else - gh api -X POST "repos/${REPO}/git/refs" \ - -f ref="refs/tags/${tag}" -f sha="$GITHUB_SHA" >/dev/null - echo "created ${tag} at ${GITHUB_SHA}" - fi - - prerelease-publish-rust: - name: Dispatch prerelease Rust publish - needs: [classify, prerelease-build-sql, prerelease-build-docs] - if: ${{ needs.classify.outputs.mode == 'prerelease' }} - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - actions: write - contents: write # create/update the release/eql- branch ref - steps: - - name: Dispatch release-plz.yml at the release commit - # release-plz refuses detached HEADs ("cannot determine current - # branch"), so a tag dispatch does not work. To still publish from the - # EXACT commit the SQL + npm artifacts shipped from — even if eql_v3 - # has advanced since the marker push — pin a release/eql- - # branch at this run's commit and dispatch against that. The branch is - # a stable pointer per identity: force-updated on retries of the same - # identity, never reused across identities. - env: - GH_TOKEN: ${{ github.token }} - VERSION: ${{ needs.classify.outputs.version }} - REPO: ${{ github.repository }} - SHA: ${{ github.sha }} - run: | - set -euo pipefail - branch="release/eql-${VERSION}" - if ! gh api -X POST "repos/${REPO}/git/refs" \ - -f ref="refs/heads/${branch}" -f sha="$SHA" >/dev/null 2>&1; then - gh api -X PATCH "repos/${REPO}/git/refs/heads/${branch}" \ - -f sha="$SHA" -F force=true >/dev/null - fi - # --repo is required: this job has no checkout, and without it gh - # tries to resolve the workflow through a local git repo and dies - # with "not a git repository". - gh workflow run release-plz.yml --repo "$REPO" --ref "$branch" - - summary: - name: Summary - runs-on: ubuntu-latest - needs: [classify, release, build-sql, build-docs, build-image, prerelease-build-sql, prerelease-build-docs, prerelease-publish-npm, prerelease-publish-rust] - if: always() - steps: - - name: Emit run summary - env: - MODE: ${{ needs.classify.outputs.mode }} - VERSION: ${{ needs.classify.outputs.version }} - CLASSIFY_RESULT: ${{ needs.classify.result }} - RELEASE_RESULT: ${{ needs.release.result }} - BUILD_SQL_RESULT: ${{ needs.build-sql.result }} - BUILD_DOCS_RESULT: ${{ needs.build-docs.result }} - BUILD_IMAGE_RESULT: ${{ needs.build-image.result }} - PRE_BUILD_SQL_RESULT: ${{ needs.prerelease-build-sql.result }} - PRE_BUILD_DOCS_RESULT: ${{ needs.prerelease-build-docs.result }} - PRE_PUBLISH_NPM_RESULT: ${{ needs.prerelease-publish-npm.result }} - PRE_PUBLISH_RUST_RESULT: ${{ needs.prerelease-publish-rust.result }} - SERVER_URL: ${{ github.server_url }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} - run: | - set -euo pipefail - { - echo "## Release result" - echo "" - echo "- mode: \`${MODE}\`" - echo "- version: \`${VERSION}\`" - echo "- classify: ${CLASSIFY_RESULT} | release: ${RELEASE_RESULT} | build-sql: ${BUILD_SQL_RESULT} | build-docs: ${BUILD_DOCS_RESULT} | build-image: ${BUILD_IMAGE_RESULT} | prerelease-build-sql: ${PRE_BUILD_SQL_RESULT} | prerelease-build-docs: ${PRE_BUILD_DOCS_RESULT} | prerelease-publish-npm: ${PRE_PUBLISH_NPM_RESULT} | prerelease-publish-rust: ${PRE_PUBLISH_RUST_RESULT}" - echo "" - echo "Run: ${SERVER_URL}/${REPOSITORY}/actions/runs/${RUN_ID}" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/packages/eql/release-plz.toml b/packages/eql/release-plz.toml index 5d00d2582..19c9d534f 100644 --- a/packages/eql/release-plz.toml +++ b/packages/eql/release-plz.toml @@ -1,17 +1,28 @@ # release-plz configuration. Mirrors the canonical cipherstash-suite setup. # -# Scope: this repository's primary artefact is the EQL SQL surface, released -# manually via `eql-` git tags (see `.github/workflows/release-eql.yml` -# and the "Cutting a release" section of CLAUDE.md). release-plz is scoped to -# the one Rust crate we ship to crates.io: `eql-bindings`. Every other workspace -# member is `publish = false` in its Cargo.toml, so release-plz skips it -# automatically (same convention as cipherstash-suite) — no per-package config. +# Scope: release-plz is scoped to the one Rust crate shipped to crates.io, +# `eql-bindings`. Every other member of this workspace is `publish = false` in +# its Cargo.toml, so release-plz skips it automatically — no per-package config. +# +# THE PATHS IN THIS FILE ARE RELATIVE TO THIS DIRECTORY, not to the repository +# root. This subtree was imported into cipherstash/stack at `packages/eql`, so +# `/.github/workflows/release-plz.yml` passes +# `--manifest-path packages/eql/Cargo.toml --config packages/eql/release-plz.toml`. +# `changelog_config` below is only read by the `update` / `release-pr` commands, +# which this repository never runs: versioning is owned by CHANGESETS +# (`scripts/sync-lockstep-versions.mjs` bumps this crate in lockstep with the +# npm package), and the workflow runs `command: release` — publish only. +# +# This header used to point at `.github/workflows/release-eql.yml` and a +# "Cutting a release" section of CLAUDE.md. Neither existed even upstream: the +# workflow was `release.yml`, and this repository has no such CLAUDE.md section. # # The crate is tagged `eql-bindings-v` (release-plz's default workspace -# tag). That tag deliberately does NOT match the EQL SQL-release automation: the -# `eql`-keyed workflows (release-eql, release-postgres-eql-image, rebuild-docs) -# exclude `eql-bindings-*`, so a crate release never triggers an SQL build, -# Docker image, or docs-site rebuild. +# tag). That tag shape is part of the release identity, not a default nobody +# chose: it deliberately does NOT match the `eql-` SQL-release +# automation, so a crate release never fires an SQL build, a Docker image, or +# the docs-site rebuild. `/.github/workflows/rebuild-docs.yml` +# carries the matching `!eql-bindings-*` exclusion. [workspace] changelog_config = "cliff.toml" diff --git a/scripts/__tests__/eql-pipeline-armed.test.mjs b/scripts/__tests__/eql-pipeline-armed.test.mjs new file mode 100644 index 000000000..90af29fe5 --- /dev/null +++ b/scripts/__tests__/eql-pipeline-armed.test.mjs @@ -0,0 +1,161 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + EQL_PACKAGE, + eqlPipelineArmed, + frozenReason, +} from '../eql-pipeline-armed.mjs' +import { FROZEN_PUBLISHERS } from '../release-gate.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * The switch that keeps the EQL release pipeline inert, and the wiring that + * makes it mean anything. + * + * The whole point of deriving the switch from `FROZEN_PUBLISHERS` is that the + * Phase-5 cutover arms the pipeline by deleting a map entry it has to delete + * anyway — the release gate refuses every release until it does. What that + * buys is undone completely by a job that stops reading the answer, and a + * dropped `if:` is invisible: while the pipeline is inert for OTHER reasons + * (the gate blocks, and `changeset publish` finds nothing), a job with no guard + * behaves exactly like a job with one. It would first differ on the day of the + * cutover, in the direction that publishes. + * + * So the wiring is asserted as an equality here, not just the script's logic. + */ + +const SCRIPT = join(REPO_ROOT, 'scripts/eql-pipeline-armed.mjs') + +/** + * Every job that must be gated on the switch, as ` / `. + * + * `release-postgres-eql-image.yml / promote-latest` is deliberately absent: it + * `needs:` two jobs that ARE gated, so it cannot run without them, and its own + * `if:` carries the floating-tag policy instead. Adding a second condition + * there would read as belt-and-braces and would in fact be a second place to + * forget. + */ +const GATED_JOBS = [ + '.github/workflows/release-plz.yml / release', + '.github/workflows/release-postgres-eql-image.yml / build-images', + '.github/workflows/release-postgres-eql-image.yml / build-sql', + '.github/workflows/release.yml / eql-docs', + '.github/workflows/release.yml / eql-image', + '.github/workflows/release.yml / eql-sql', + '.github/workflows/release.yml / prerelease-eql-crate', + '.github/workflows/release.yml / prerelease-eql-docs', + '.github/workflows/release.yml / prerelease-eql-npm', + '.github/workflows/release.yml / prerelease-eql-sql', +] + +/** The workflows that run the script, each exposing it as an `armed` output. */ +const SWITCH_WORKFLOWS = [ + '.github/workflows/release.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', +] + +/** The condition text of every job in a workflow, keyed ` / `. */ +function jobConditions(relPath) { + const wf = readWorkflow(relPath) + return Object.entries(wf?.jobs ?? {}).map(([name, job]) => [ + `${relPath} / ${name}`, + String(job?.if ?? '').replace(/\s+/g, ' '), + ]) +} + +function run(env = {}) { + return execFileSync('node', [SCRIPT], { + cwd: REPO_ROOT, + encoding: 'utf8', + env: { ...process.env, ...env }, + }) +} + +describe('the switch answers from FROZEN_PUBLISHERS', () => { + it('is inert while the package is frozen', () => { + expect(eqlPipelineArmed(new Map([[EQL_PACKAGE, 'a reason']]))).toBe(false) + }) + + it('is armed once the entry is gone — the cutover, exercised now', () => { + // The state that does not exist yet. Without this the "armed" branch would + // first execute during the cutover, which is the worst possible moment to + // discover the switch was inverted. + expect(eqlPipelineArmed(new Map())).toBe(true) + expect(eqlPipelineArmed(new Map([['@cipherstash/other', 'x']]))).toBe(true) + }) + + it('reports the reason from the map, not from a sentence of its own', () => { + expect(frozenReason(new Map([[EQL_PACKAGE, 'because']]))).toBe('because') + expect(frozenReason(new Map())).toBeNull() + }) + + it('matches the live map, whichever state that is in', () => { + // Not "is currently false". The map is what Phase 5 edits, and pinning the + // verdict here would make the cutover fail in this file for no reason. + expect(eqlPipelineArmed()).toBe(!FROZEN_PUBLISHERS.has(EQL_PACKAGE)) + }) +}) + +describe('the switch is readable by a workflow', () => { + it('writes `armed=` to GITHUB_OUTPUT', () => { + const dir = mkdtempSync(join(tmpdir(), 'eql-armed-')) + const outputFile = join(dir, 'output') + try { + execFileSync('sh', ['-c', `: > "${outputFile}"`]) + run({ GITHUB_OUTPUT: outputFile }) + expect(readFileSync(outputFile, 'utf8')).toBe( + `armed=${eqlPipelineArmed()}\n`, + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('says which state it is in, and why, on stdout', () => { + const stdout = run() + expect(stdout).toContain(EQL_PACKAGE) + expect(stdout).toContain(eqlPipelineArmed() ? 'ARMED' : 'INERT') + if (!eqlPipelineArmed()) expect(stdout).toContain(frozenReason()) + }) + + it('does not write GITHUB_OUTPUT when there is none', () => { + // A `main()` that assumed the variable would throw on a local run, which is + // how someone checks the answer before a cutover. + const stdout = run({ GITHUB_OUTPUT: '' }) + expect(stdout).toContain(EQL_PACKAGE) + }) +}) + +describe('every EQL publish job reads it', () => { + it('exposes the switch as a job output in each release workflow', () => { + const missing = SWITCH_WORKFLOWS.filter((relPath) => { + const job = readWorkflow(relPath)?.jobs?.['eql-armed'] + const runsIt = (job?.steps ?? []).some((step) => + String(step?.run ?? '').includes('scripts/eql-pipeline-armed.mjs'), + ) + return !(runsIt && job?.outputs?.armed) + }) + expect( + missing, + 'These workflows publish an EQL artefact and no longer compute the arming switch, so nothing downstream can be gated on it.', + ).toEqual([]) + }) + + it('gates exactly the jobs that publish an EQL artefact', () => { + const gated = SWITCH_WORKFLOWS.flatMap(jobConditions) + .filter(([, condition]) => + condition.includes("needs.eql-armed.outputs.armed == 'true'"), + ) + .map(([id]) => id) + + expect( + gated.sort(), + "The set of jobs gated on the EQL arming switch has changed. A job that LOST its guard will publish on the day the FROZEN_PUBLISHERS entry is deleted, whether or not anyone meant it to — and until then it is indistinguishable from a job that still has one. Spell the guard exactly `needs.eql-armed.outputs.armed == 'true'`.", + ).toEqual(GATED_JOBS) + }) +}) diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 22c16e62a..c0388db8f 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -865,10 +865,12 @@ const CI_EXEMPT_CARGO_TASKS = new Map([ 'test:lint', '`cargo fmt --check` scoped to tests/sqlx. `test:crates` runs `cargo fmt --check` at the EQL workspace ROOT, and tests/sqlx is a workspace member, so the CI run covers exactly these files and four crates more — verified against `cargo fmt --check -v`, which lists packages/eql/tests/sqlx/** among its targets. test-eql.yml says the same thing at the `rust-crates` job, which is where the standalone lint step used to be.', ], - [ - 'docs:generate:json', - 'Generates docs/api/json/eql-manifest.json for the release docs bundle. Its only caller is `packages/eql/.github/workflows/_build-docs.yml` — one of the four unported RELEASE workflows in UNPORTED_DEPOSIT — so this is category (d): it becomes reachable the day that file is ported, and this entry then goes stale and must be deleted. It is also not portable as-is, since it falls back to `mise run docs:generate`, which needs a doxygen binary no job installs. Its cargo step (`cargo run -p eql-codegen dump-catalog`) is separately exercised in CI by `test:matrix:catalog-coverage`.', - ], + // `docs:generate:json` WAS HERE, as category (d) — "unreached only because the + // workflow that runs it has not been ported yet" — naming + // `packages/eql/.github/workflows/_build-docs.yml` as its sole caller. That + // file is now `.github/workflows/_build-eql-docs.yml` and does run, so the + // exemption went stale exactly as it said it would, and the staleness check + // below is what forced its removal rather than a memory of having written it. ]) /** Cargo tasks a given reachability set leaves unaccounted for. */ @@ -1292,6 +1294,13 @@ describe('every cargo check EQL owns is reached by a root workflow', () => { * actually fails. */ const SOLE_CALLER_WORKFLOWS = [ + // docs:generate:json. It arrived on this list the day the release port + // landed — until then that task was reachable from nothing at all and carried + // a written exemption saying so. Note what "sole caller" costs here that it + // does not cost for the three below: this workflow runs only on a RELEASE, so + // a break in the doxygen -> JSON pipeline surfaces mid-publish rather than on + // the pull request that caused it. + `${WORKFLOW_DIR}/_build-eql-docs.yml`, `${WORKFLOW_DIR}/bench-eql.yml`, // test:bench `${WORKFLOW_DIR}/macro-expand-eql.yml`, // test:matrix:expand `${WORKFLOW_DIR}/test-eql.yml`, // most of the suite @@ -1350,63 +1359,81 @@ describe('the guard fails when a workflow stops calling a cargo check', () => { }) /** - * What is still allowed to sit in the deposited `.github`, and why. + * The deposited `.github` is GONE, and must stay gone. + * + * This used to be a SHRINKING ALLOWLIST — eleven entries, asserted by equality, + * emptied one commit at a time as each file was ported or deliberately dropped. + * The last of them left with the release port, which is what the allowlist's own + * failure message asked for: "When this list empties, delete the directory and + * replace this check with `expect(existsSync(...)).toBe(false)`." + * + * The rule it enforced does not expire with it. A workflow file under a package + * reads as live CI and is not: GitHub reads workflows from the repository root + * alone, so anything deposited here executes on no event — which is + * indistinguishable, on every run and every PR page, from a check that passes. + * That is the failure this whole file exists to prevent, and `lintWiring.test.ts` + * asserts exactly the same thing for protect-ffi's deposit. * - * The end state is no directory at all — the rule `lintWiring.test.ts` applies - * to protect-ffi's deposit, and for the same reason: a workflow file under a - * package reads as live CI and is not. It cannot be deleted in one step here, - * because four of EQL's ten workflows are the RELEASE machinery. Porting those - * to the repository root is what makes them fire, and it is gated on repointing - * npm and crates.io trusted publishing — an irreversible cutover that has not - * happened. Deleting them first would mean porting from git history at the one - * moment nobody wants to be reconstructing a publish pipeline. + * WHERE EACH FILE WENT, because "it was deleted" is not the same claim as "it + * was ported" and a reader a year from now cannot tell them apart from a diff: * - * So this is a SHRINKING allowlist, not an exemption list. Each entry names a - * file that has not been ported yet; porting one means deleting it here in the - * same commit. The equality below fails in both directions — a file that comes - * back fails, and so does an entry left behind after its file is gone, which is - * what turns the last removal into "delete the directory" rather than a check - * that quietly stops meaning anything. + * _build-sql.yml -> .github/workflows/_build-eql-sql.yml + * _build-docs.yml -> .github/workflows/_build-eql-docs.yml + * release.yml -> merged into .github/workflows/release.yml + * (npm trusted publishing binds to a + * filename, so it could not stay separate) + * release-plz.yml -> .github/workflows/release-plz.yml, name + * unchanged (crates.io binds to it) + * release-postgres-eql-image.yml -> same filename at the root + * rebuild-docs.yml -> merged into the root file of that name + * lint-release.yml -> merged into the root file of that name + * .github/actionlint.yaml -> DROPPED: the root copy already carries the + * `blacksmith-16vcpu-ubuntu-2204` label + * ISSUE_TEMPLATE/docs-feedback.yml -> DROPPED: the root copy is the same file + * with the Code of Conduct link already + * repointed at cipherstash/stack + * .github/release.yml -> DROPPED: GitHub's release-notes + * categoriser. Nothing here generates notes + * that way — changesets writes CHANGELOGs + * and the FFI release passes `--notes` + * explicitly — so it would be configuration + * that does nothing + * workflows/README.md -> DROPPED: it documented a merge-queue model + * this repository does not have. `main` is + * unprotected and there is no queue (see + * test-eql.yml's header, which checked that + * against the live API), so porting it would + * have installed a confidently wrong + * document at the root. Its still-true half + * — the two release tag families — is in + * AGENTS.md and in rebuild-docs.yml's header */ -const UNPORTED_DEPOSIT = [ - // Phase 5 — the release cutover. Each publishes something, and each is inert - // until trusted publishing is repointed at cipherstash/stack. - '.github/release.yml', // release-notes categorisation config, not a workflow - '.github/workflows/README.md', // documents the four below - '.github/workflows/_build-docs.yml', // the sole caller of `docs:generate:json` - '.github/workflows/_build-sql.yml', - '.github/workflows/lint-release.yml', // actionlints the four below - '.github/workflows/rebuild-docs.yml', - '.github/workflows/release-plz.yml', - '.github/workflows/release-postgres-eql-image.yml', - '.github/workflows/release.yml', - // Not CI. Ported with the repository settings, not with a workflow. - '.github/ISSUE_TEMPLATE/docs-feedback.yml', - '.github/actionlint.yaml', -].sort() - -describe('the imported workflow directory is on its way out', () => { - it('holds only the files still waiting to be ported', () => { - const deposit = existsSync(join(REPO_ROOT, DEAD_GITHUB_DIR)) - ? execFileSync('git', ['ls-files', '-z', '--', DEAD_GITHUB_DIR], { - cwd: REPO_ROOT, - encoding: 'utf8', - }) - .split('\0') - .filter(Boolean) - .map((path) => path.slice('packages/eql/'.length)) - .sort() - : [] +describe('the imported workflow directory is gone', () => { + it('does not exist', () => { + expect( + existsSync(join(REPO_ROOT, DEAD_GITHUB_DIR)), + `${DEAD_GITHUB_DIR} is back. GitHub reads workflows from the repository root alone, ` + + 'so anything in there runs on no event — which looks exactly like a check that passes. ' + + 'Put the workflow in .github/workflows/ instead.', + ).toBe(false) + }) + + it('has no tracked files, even if the directory is untracked scratch', () => { + // `existsSync` alone would miss the case that actually matters: a file + // committed back into the deposit by a subtree pull, in a working tree + // where the directory was never removed locally. + const tracked = execFileSync( + 'git', + ['ls-files', '-z', '--', DEAD_GITHUB_DIR], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ) + .split('\0') + .filter(Boolean) expect( - deposit, - `${DEAD_GITHUB_DIR} no longer matches the list of files still waiting to be ported.\n` + - 'If you PORTED one, delete it from the deposit and from UNPORTED_DEPOSIT in the same commit — ' + - 'leaving it here means two copies of a workflow, one of which GitHub ignores.\n' + - 'If you ADDED one, it is inert: GitHub reads workflows from the repository root alone.\n' + - 'When this list empties, delete the directory and replace this check with ' + - '`expect(existsSync(...)).toBe(false)`.', - ).toEqual(UNPORTED_DEPOSIT) + tracked, + `These files are tracked under ${DEAD_GITHUB_DIR} and execute on nothing.`, + ).toEqual([]) }) }) @@ -1456,6 +1483,50 @@ function reachesCargo(name, seen = new Set()) { return result } +/** + * The workflows that MUST NOT restore a cache, read from the gate that says so. + * + * THE TWO RULES IN THIS REPOSITORY POINT OPPOSITE WAYS, and the EQL release port + * is where they met. `scripts/lint-no-workflow-caching.mjs` forbids a GitHub + * Actions cache restore anywhere an artefact gets published — a poisoned entry + * would execute in the job holding the npm or crates.io credential — and + * `Swatinem/rust-cache` is precisely such a restore. The check below asks for + * one on every job that compiles Rust. Four release jobs do both: they compile + * the EQL workspace (`mise run build` shells out to `cargo run -p eql-codegen`) + * inside a workflow that publishes. + * + * Supply chain wins, so those jobs pay a cold compile. What must not happen is + * this file and that linter disagreeing about WHICH jobs those are: a workflow + * added to one list and not the other would either be told to add a cache the + * linter then rejects, or be quietly dropped from both checks. + * + * So the list is derived rather than copied. The linter prints its default + * TARGETS on a clean run — the same trick `lint-no-workflow-caching.test.mjs` + * uses to read them — because the script lints on import and cannot be imported + * for the constant alone. If it ever exits non-zero this throws, rather than + * falling back to an empty set: an exemption list that silently empties would + * re-report all four jobs, and one that silently fills would excuse everything. + */ +const NO_CACHE_WORKFLOWS = (() => { + const stdout = execFileSync( + 'node', + [join(REPO_ROOT, 'scripts/lint-no-workflow-caching.mjs')], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ) + const targets = stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith(`${WORKFLOW_DIR}/`)) + if (targets.length === 0) { + throw new Error( + 'lint-no-workflow-caching.mjs printed no targets. Its success output is the ' + + 'only readable copy of its TARGETS, and without it the publish-path ' + + 'exemption below silently covers nothing.', + ) + } + return new Set(targets) +})() + /** * Every job in a root workflow, with the cargo-reaching mise tasks its steps * invoke and whether it restores the Rust cache. @@ -1512,8 +1583,36 @@ describe('every job that compiles Rust restores the shared cache', () => { ).toBeGreaterThan(0) }) + it('exempts the publish paths, and only those', () => { + // The exemption is worth an assertion of its own rather than only ever + // appearing as an absence in the check below: if the derivation above stops + // selecting the release jobs, they rejoin that check and the natural repair + // is to add a rust-cache step the caching gate then rejects. + const exempt = RUST_JOBS.filter(({ relPath }) => + NO_CACHE_WORKFLOWS.has(relPath), + ) + expect( + exempt.map(({ relPath, jobName }) => `${relPath} (${jobName})`).sort(), + 'No cargo-compiling job sits in a no-cache publish workflow any more. If the EQL release pipeline moved, move this with it; if it was deleted, delete this.', + ).toEqual([ + '.github/workflows/_build-eql-docs.yml (publish-docs)', + '.github/workflows/_build-eql-sql.yml (build)', + '.github/workflows/release-postgres-eql-image.yml (build-sql)', + '.github/workflows/release.yml (prerelease-eql-npm)', + ]) + + // …and none of them smuggled the cache in anyway. The caching gate would + // catch it too; this says so where the reader is already looking. + expect( + exempt.filter(({ cached }) => cached).map(({ relPath }) => relPath), + 'A publish-path job carries a `Swatinem/rust-cache` step. That is a GitHub Actions cache restore inside a job holding a publishing credential — see scripts/lint-no-workflow-caching.mjs.', + ).toEqual([]) + }) + it('leaves no cargo-compiling job uncached', () => { - const uncached = RUST_JOBS.filter(({ cached }) => !cached).map( + const uncached = RUST_JOBS.filter( + ({ relPath, cached }) => !cached && !NO_CACHE_WORKFLOWS.has(relPath), + ).map( ({ relPath, jobName, compiling }) => `${relPath} (${jobName}): runs ${compiling.join(', ')}`, ) diff --git a/scripts/__tests__/lint-no-workflow-caching.test.mjs b/scripts/__tests__/lint-no-workflow-caching.test.mjs index e0fb11f34..fbda6351c 100644 --- a/scripts/__tests__/lint-no-workflow-caching.test.mjs +++ b/scripts/__tests__/lint-no-workflow-caching.test.mjs @@ -17,6 +17,14 @@ import { REPO_ROOT } from './lib/repo-root.mjs' const TARGET_WORKFLOWS = [ '.github/workflows/release.yml', '.github/workflows/_build-ffi-artifacts.yml', + // The EQL release line. `scripts/__tests__/eql-suite-ci.test.mjs` reads the + // script's real list too, and uses it to exempt these workflows from the + // rust-cache requirement — the two rules point opposite ways for a job that + // compiles Rust inside a publish path, and that file records which one wins. + '.github/workflows/_build-eql-sql.yml', + '.github/workflows/_build-eql-docs.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', '.github/workflows/tests-supply-chain.yml', ] diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index 3fc7dee8d..2d3b4f680 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -65,6 +65,49 @@ const EXPECTED_DISPATCHABLE = [ '.github/workflows/lint-release.yml', '.github/workflows/osv-scanner.yml', '.github/workflows/tests-rust.yml', + // The EQL release line. `release.yml` gained a dispatch with the port, + // because the EQL PRERELEASE path is cut by dispatching a batching branch + // whose HEAD is a `chore(release):` marker; the other two are how an image or + // a crate publish is re-run by hand after a partial failure. + '.github/workflows/release.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', +] + +/** + * Jobs that legitimately do NOT run on a plain manual dispatch, with the reason. + * + * WHY THIS EXISTS AT ALL. The check below reads "a declared `workflow_dispatch` + * must actually dispatch", and it enforced that by requiring EVERY job-level + * `if:` in such a workflow to be true under one synthetic dispatch context. + * That is exact for a workflow with one path through it, which every + * dispatchable workflow here had. `release.yml` has two, and they are mutually + * exclusive by construction: `classify` reports `production` for `main` and + * `prerelease` for a marker commit on any other branch, and each half of the + * file is keyed on one of them. Under any single context one half is false. No + * spelling of the conditions avoids that, because the exclusivity is the + * design. + * + * WHAT IS NOT WEAKENED. The alternative was to relax the rule to "at least one + * job runs", which would still have caught the original defect (a + * single-job workflow whose only job skipped) and would have said nothing about + * a ten-job workflow where nine skip. This keeps the per-job requirement and + * makes the exceptions an equality instead — so an unlisted job that stops + * running on dispatch fails, AND an entry that stops being needed fails. + * + * The synthetic context dispatches against `refs/heads/main`, so "production" + * is the path being modelled. That is the right default: it is what a dispatch + * of `release.yml` does unless someone deliberately points it at a release + * branch. + */ +const DISPATCH_SKIPPED_JOBS = [ + // The four prerelease jobs. A dispatch against `main` classifies as + // `production`, so these are correctly skipped; they run when the dispatch + // names a branch whose HEAD commit subject is `chore(release):`. + '.github/workflows/release.yml / prerelease-eql-crate', + '.github/workflows/release.yml / prerelease-eql-docs', + '.github/workflows/release.yml / prerelease-eql-npm', + '.github/workflows/release.yml / prerelease-eql-sql', ] /** @@ -373,6 +416,30 @@ function runsWhen(condition, context) { */ const PERMISSIVE_NEEDS = { changes: { outputs: { relevant: 'true' } }, + // release.yml, release-plz.yml and release-postgres-eql-image.yml. Every one + // of these is a gate that has nothing to do with how the run was triggered, + // so it is held open — otherwise "this job skips because there is nothing to + // publish" would be indistinguishable from "this job skips on a dispatch", + // and the check below would pass for the wrong reason on the FFI jobs it has + // been covering since before the EQL port. + classify: { outputs: { mode: 'production', version: '3.0.6' } }, + 'eql-armed': { outputs: { armed: 'true' } }, + gate: { result: 'success', outputs: { ffi: 'true' } }, + 'publish-ffi': { result: 'success' }, + release: { + result: 'success', + outputs: { + eql_published: 'true', + eql_version: '3.0.6', + // A FINAL, not a prerelease: `eql-image` runs only for a final, and this + // is the value that lets it through. The prerelease half of the file is + // in DISPATCH_SKIPPED_JOBS rather than being served by a second context. + eql_prerelease: 'false', + }, + }, + // release-postgres-eql-image.yml's `promote-latest` reads this from its own + // `build-sql` job, which copies the dispatch input through. + 'build-sql': { outputs: { update_floating_tags: 'true' } }, } /** @@ -512,7 +579,24 @@ describe('a declared workflow_dispatch actually dispatches', () => { ).not.toEqual([]) }) - for (const { id, condition } of DISPATCHABLE_CONDITIONS) { + it('skips exactly the jobs declared unreachable by a plain dispatch', () => { + // An equality, both directions. A job that quietly stops running on + // dispatch fails; an entry left behind after its job starts running (or + // after the job is deleted) fails too, so the list cannot become a place + // findings go to be forgotten. + const skipped = DISPATCHABLE_CONDITIONS.filter( + ({ condition }) => !runsWhen(condition, CONTEXTS.workflow_dispatch), + ).map((entry) => entry.id) + + expect( + skipped.sort(), + 'The set of jobs that do not run on a manual dispatch has changed. If a job JOINED it, its `workflow_dispatch:` just became decorative for that job — gate on the case that genuinely cannot run rather than enumerating the events that can. If a job LEFT it, delete its DISPATCH_SKIPPED_JOBS entry in the same commit.', + ).toEqual([...DISPATCH_SKIPPED_JOBS].sort()) + }) + + for (const { id, condition } of DISPATCHABLE_CONDITIONS.filter( + (entry) => !DISPATCH_SKIPPED_JOBS.includes(entry.id), + )) { it(`${id} runs on a manual dispatch`, () => { expect( runsWhen(condition, CONTEXTS.workflow_dispatch), diff --git a/scripts/__tests__/workflow-publish-permissions.test.mjs b/scripts/__tests__/workflow-publish-permissions.test.mjs index cd4decc8b..a7fff8be0 100644 --- a/scripts/__tests__/workflow-publish-permissions.test.mjs +++ b/scripts/__tests__/workflow-publish-permissions.test.mjs @@ -46,6 +46,56 @@ const OIDC_JOBS = [ '.github/workflows/release.yml / publish-ffi', // `changeset publish` for the JS packages, plus the Version Packages PR. '.github/workflows/release.yml / release', + // The EQL prerelease path publishes @cipherstash/eql directly rather than + // through changesets, so it mints its own npm token. + '.github/workflows/release.yml / prerelease-eql-npm', + // release-plz publishes the eql-bindings crate to crates.io, which uses the + // same OIDC token exchange. A DIFFERENT registry, and the reason this list + // could not stay a two-line one: the workflow filename is bound at crates.io + // rather than npm, but the scope is the same scope. + '.github/workflows/release-plz.yml / release', +] + +/** + * The jobs in a publishing workflow that may hold ANY writable scope, as + * ` / `. A superset of `OIDC_JOBS`, asserted as such below. + * + * WHY THIS IS A SECOND LIST AND NOT THE SAME ONE. The third check below reads + * "a job in a publishing workflow that does not publish must not be able to + * write to the repository", and it enforced that by asking whether the job was + * in `OIDC_JOBS` — which made the npm-credential list do double duty as the + * write list. That was exactly right while the only writing jobs were the two + * that publish. It stopped being expressible the moment `release.yml` grew the + * EQL release line: `eql-sql` creates the `eql-` GitHub release, + * `eql-image` dispatches another workflow, `prerelease-eql-crate` moves a + * branch ref. All three need a writable scope; none of them may mint a publish + * token, and putting them in `OIDC_JOBS` to buy the write would have said they + * could. + * + * So the two facts are separated. `OIDC_JOBS` still answers "who may publish?" + * as an equality, which is the strong claim; this answers the weaker one, and + * every entry names the scope and what it is for — because `contents: write` in + * a file registered as a trusted publisher is still the scope that lets a + * compromised step rewrite the tree the publish jobs build from. + */ +const REPO_WRITE_JOBS = [ + ...OIDC_JOBS, + // contents: write — creates the eql- tag and GitHub release, and + // uploads the SQL bundle to it. Reached through a job-level `uses:`, whose + // grant is the CEILING for the reusable workflow, so this is also what + // constrains `_build-eql-sql.yml`. + '.github/workflows/release.yml / eql-sql', + // contents: write — attaches the docs bundle to the release above. + '.github/workflows/release.yml / eql-docs', + // actions: write — `gh workflow run release-postgres-eql-image.yml`. + '.github/workflows/release.yml / eql-image', + // contents: write — the prerelease halves of the two above. + '.github/workflows/release.yml / prerelease-eql-sql', + '.github/workflows/release.yml / prerelease-eql-docs', + // actions: write to dispatch release-plz.yml, contents: write to pin the + // release/eql- branch it must be dispatched against (release-plz + // refuses a detached HEAD). + '.github/workflows/release.yml / prerelease-eql-crate', ] /** @@ -107,6 +157,9 @@ const workflows = workflowFiles().map((file) => { /** Is this the ` / ` of a job sanctioned to publish? */ const sanctioned = (file, name) => OIDC_JOBS.includes(`${file} / ${name}`) +/** …and of a job sanctioned to hold a writable scope at all? */ +const mayWrite = (file, name) => REPO_WRITE_JOBS.includes(`${file} / ${name}`) + describe('supply chain — a publishing workflow grants OIDC per job', () => { it('discovers the jobs that hold id-token: write', () => { // The guard on the scan. Every check below is "for each workflow that mints @@ -156,7 +209,7 @@ describe('supply chain — a publishing workflow grants OIDC per job', () => { ) .flatMap(({ file, workflowLevel, jobs }) => jobs - .filter(([name]) => !sanctioned(file, name)) + .filter(([name]) => !mayWrite(file, name)) .flatMap(([name, job]) => { const writes = writable(effective(job, workflowLevel)) return writes.length @@ -166,10 +219,22 @@ describe('supply chain — a publishing workflow grants OIDC per job', () => { ) expect( offenders, - 'A job in a publishing workflow that does not publish must not be able to write to the repository.', + 'A job in a publishing workflow that does not publish must not be able to write to the repository. If it genuinely needs a writable scope and must NOT be able to publish, add it to REPO_WRITE_JOBS with the scope and the reason — not to OIDC_JOBS.', ).toEqual([]) }) + it('never sanctions a write without sanctioning it as a write', () => { + // The one way the split above could go wrong: a publisher added to + // `OIDC_JOBS` and not carried into `REPO_WRITE_JOBS`. It is spelled as a + // spread today, so this cannot fail — which is the point. It fails the day + // somebody writes the two lists out separately, before the third check + // starts reporting a publisher as an offender. + const missing = OIDC_JOBS.filter( + (entry) => !REPO_WRITE_JOBS.includes(entry), + ) + expect(missing).toEqual([]) + }) + it('declares workflow-level permissions in a publishing workflow', () => { // Absent is not read-only: with no `permissions:` key at all, jobs fall back // to the REPOSITORY default, which is settings-controlled and outside this diff --git a/scripts/eql-pipeline-armed.mjs b/scripts/eql-pipeline-armed.mjs new file mode 100644 index 000000000..a00d36eff --- /dev/null +++ b/scripts/eql-pipeline-armed.mjs @@ -0,0 +1,93 @@ +/** + * Is the EQL release pipeline allowed to publish anything from THIS repository? + * + * ## Why this is a script and not a workflow input + * + * The whole EQL release pipeline — `release.yml`'s EQL jobs, `release-plz.yml`, + * `_build-eql-sql.yml`, `_build-eql-docs.yml`, + * `release-postgres-eql-image.yml` — was built here while `@cipherstash/eql` is + * still published from `cipherstash/encrypt-query-language`. It exists so it + * can be reviewed, dry-run and corrected, and it must reach no registry until + * the Phase-5 cutover repoints trusted publishing. That is a state, and a state + * needs a switch. + * + * The obvious switch is a boolean somebody flips. It is the wrong one, for the + * reason `scripts/release-gate.mjs` records about its own map: an entry left + * behind does not fail on the day it goes wrong, it fails on the next release. + * A hand-flipped flag has the same shape in reverse — the cutover PR repoints + * npm and crates.io, and NOTHING fails if it forgets the flag. The pipeline + * simply stays inert, and the first EQL release after the cutover quietly + * publishes an npm package with no SQL release, no docs and no crate. + * + * So the switch is DERIVED from the fact it depends on. `FROZEN_PUBLISHERS` in + * `scripts/release-gate.mjs` is the single written record of "this package + * lives here and is published elsewhere", and the cutover deletes its entry + * because the release gate blocks every release until it does. Deleting it is + * what arms this pipeline. There is no second edit to forget. + * + * ## Why not just use the release gate + * + * `release-gate.mjs` answers a REGISTRY question — which committed versions are + * missing from npm — and it already blocks a frozen package's release by + * exiting non-zero. That covers `release.yml`, whose EQL jobs all sit behind + * `needs: [gate]`. + * + * It does not cover `release-plz.yml`. That workflow publishes a CRATE, on its + * own `push: main` trigger, and crates.io is not a registry the gate models at + * all. Keying it on the npm answer would also be a race: on the push that + * releases version V, `release.yml` and `release-plz.yml` start together, and + * whichever npm publish finishes first would flip the other's answer from + * "missing" to "published" mid-run — so the crate would be skipped on exactly + * the push that was supposed to release it. + * + * This asks a question with no registry in it and no clock in it: is + * `@cipherstash/eql` still recorded as published from somewhere else? Same + * answer for every job in every workflow on every run. + */ +import { appendFileSync } from 'node:fs' +import process from 'node:process' +import { fileURLToPath } from 'node:url' +import { FROZEN_PUBLISHERS } from './release-gate.mjs' + +/** The npm package whose publisher decides whether the whole line is armed. */ +export const EQL_PACKAGE = '@cipherstash/eql' + +/** + * `true` when this repository may publish the EQL release line. + * + * The map is a parameter so the tests can drive both states without mutating + * the real one — and so the "armed" case is exercised now rather than first + * being exercised by the cutover. + */ +export function eqlPipelineArmed(frozen = FROZEN_PUBLISHERS) { + return !frozen.has(EQL_PACKAGE) +} + +/** + * Why it is not armed, or `null` when it is. + * + * The reason is the `FROZEN_PUBLISHERS` entry itself rather than a sentence + * written here, so the job log says which repository still owns the publish and + * cannot drift from the map that decided it. + */ +export function frozenReason(frozen = FROZEN_PUBLISHERS) { + return frozen.get(EQL_PACKAGE) ?? null +} + +function main() { + const armed = eqlPipelineArmed() + + console.log( + armed + ? `${EQL_PACKAGE} is published from this repository — the EQL release pipeline is ARMED.` + : `${EQL_PACKAGE} is a frozen publisher — the EQL release pipeline is INERT.\n ${frozenReason()}`, + ) + + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `armed=${armed}\n`) + } +} + +// Importable without running, so the tests can read the two exports above +// without writing to GITHUB_OUTPUT. +if (process.argv[1] === fileURLToPath(import.meta.url)) main() diff --git a/scripts/lint-no-workflow-caching.mjs b/scripts/lint-no-workflow-caching.mjs index 167af61c7..093ea4b3b 100644 --- a/scripts/lint-no-workflow-caching.mjs +++ b/scripts/lint-no-workflow-caching.mjs @@ -17,6 +17,23 @@ const TARGETS = process.argv.slice(2).length // calls it too — cannot become a way to build these artifacts under // different rules. '.github/workflows/_build-ffi-artifacts.yml', + // The EQL release line. `_build-eql-sql.yml` and `_build-eql-docs.yml` + // are reached from release.yml anyway (the traversal follows a job-level + // `uses:`) and are named for the same reason `_build-ffi-artifacts.yml` + // is: so a second caller cannot become a way to build the released SQL + // and docs under different rules. + // + // `release-plz.yml` and `release-postgres-eql-image.yml` are NOT reached + // from anywhere — `release.yml` starts them with `gh workflow run`, which + // no traversal can follow — so naming them here is the only thing that + // brings them into scope. One publishes a crate to crates.io over OIDC; + // the other pushes a multi-arch image to GHCR. Both hold a publishing + // credential, which is the property this gate is about, and neither is + // reachable by reading `uses:` from a target. + '.github/workflows/_build-eql-sql.yml', + '.github/workflows/_build-eql-docs.yml', + '.github/workflows/release-plz.yml', + '.github/workflows/release-postgres-eql-image.yml', '.github/workflows/tests-supply-chain.yml', ] @@ -155,6 +172,57 @@ const AUDITED_ACTIONS = new Map([ // fires only on a truthy value, which is how a mise-action step carrying no // `cache:` passed this gate. SHA-pinned at every call site. ['jdx/mise-action', { cacheInput: 'cache' }], + + // ---- The EQL release line ------------------------------------------- + // + // Creates and attaches the `eql-` GitHub release from + // `_build-eql-sql.yml` and `_build-eql-docs.yml`. Read at the pinned v2: its + // inputs are release metadata and file globs, and it uploads through the + // releases API. No cache, no cache input. + ['softprops/action-gh-release', { cacheInput: null }], + // Imports the GPG key release-plz signs its commit and tag with. Read at the + // pinned v7: key material, git config and fingerprint inputs only. + ['crazy-max/ghaction-import-gpg', { cacheInput: null }], + // Publishes the `eql-bindings` crate. It has no cache input — AND IT IS A + // COMPOSITE, so a second sentence is owed here that the other entries do not + // need. At the pinned v0.5 it reaches three published actions of its own + // (`taiki-e/install-action`, `cargo-bins/cargo-binstall`, + // `release-plz/git-config`), and this gate CANNOT read them: it opens local + // `./` composites and nothing else. So this entry is an audit of the version + // pinned above, not a standing property of the action — read the composite + // again when the pin moves. It was read at 2eb1d8bc: the three nested actions + // download binaries (GitHub releases and crates.io) and none of them touches + // the GitHub Actions cache. + ['release-plz/action', { cacheInput: null }], + + // ---- Docker, for the postgres-eql image ------------------------------ + // + // The plan for these three said their caching was "registry-backed rather + // than GitHub-Actions-cache", so the allowlist's rule would not transfer. + // Reading the pinned action manifests, that is true of exactly ONE of them. + // + // `cache-image` DEFAULTS TO TRUE and stores the binfmt image in the GitHub + // Actions cache — the same shape as `jdx/mise-action` above, and an omitted + // key is a restore. + ['docker/setup-qemu-action', { cacheInput: 'cache-image' }], + // `cache-binary` DEFAULTS TO TRUE and caches the buildx binary the same way. + ['docker/setup-buildx-action', { cacheInput: 'cache-binary' }], + // Authenticates to GHCR. No cache, no cache input. + ['docker/login-action', { cacheInput: null }], + // THE ONE THE PLAN DESCRIBED, and the only entry that needs + // `forbiddenInputs`. It has no boolean that turns caching off, because it has + // no caching of its own to turn off: buildx caches only what `cache-from` / + // `cache-to` ask it to. Those are usually registry-backed + // (`type=registry,ref=…`), which is outside this gate's concern — but + // `type=gha` names the GitHub Actions cache directly, and would be a restore + // AND a write inside the job that pushes the published image. `cacheInput` + // cannot express that: there is no input to set to `false`. So the two inputs + // are forbidden outright. Adding one back means deciding here, in this file, + // which backend it names. + [ + 'docker/build-push-action', + { cacheInput: null, forbiddenInputs: ['cache-from', 'cache-to'] }, + ], ]) // A secondary, deliberately over-broad read of the action's name. It is NOT the @@ -307,12 +375,31 @@ function checkStep(step, at, bodyAudited = false) { // from a parallel list of per-action rules. An audited action that caches is // an additional constraint on it, not a substitute for being audited. const path = actionPath(uses) - const cacheInput = AUDITED_ACTIONS.get(path)?.cacheInput + const audited = AUDITED_ACTIONS.get(path) + const cacheInput = audited?.cacheInput if (cacheInput) { const reason = explicitFalseReason(step, cacheInput) if (reason) offenders.push(`${at}: ${path} ${reason}`) } + // The other half of the same audit, for an action whose caching is opted + // INTO rather than defaulted on. `cacheInput` asks for an explicit `false`; + // there is nothing to set to `false` here, so the requirement is that the + // input is absent. Presence is the finding, whatever its value — the value is + // a backend name (`type=gha` restores and writes the GitHub Actions cache, + // `type=registry` does not), and deciding which is which by string-matching a + // buildx cache spec would be a parser this file does not want to own. Naming + // one at all is the moment to record the decision on the allowlist entry. + for (const forbidden of audited?.forbiddenInputs ?? []) { + if (step?.with && Object.hasOwn(step.with, forbidden)) { + offenders.push( + `${at}: ${path} must not set \`${forbidden}\` — it can name the GitHub ` + + 'Actions cache (`type=gha`), and this action has no input that turns ' + + 'caching off. See its AUDITED_ACTIONS entry.', + ) + } + } + // One verdict per `uses:`, most specific first — a step reported twice reads // as two problems and gets fixed once. if (CACHE_ACTION.test(uses)) { From ab4c4f38504b0fa45daa3b5fce9127065a3694e4 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 24 Aug 2026 14:43:52 +1000 Subject: [PATCH 2/5] fix(eql): act on the review findings, and cut the comment volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven findings from the review of the ported pipeline, all confirmed before acting. Three were guards that could not fire, which is the failure mode this repository cares most about — a check that reads as protection and is incapable of reporting anything. `rebuild-docs.yml` was the worst of them, and worse than reported: it has never run, not once, against `@cipherstash/stack@1.0.0`, `1.1.0` and `1.1.1`. A ref created with GITHUB_TOKEN starts no workflow run, and changesets creates those tags with that token — the same rule this pipeline already works around for the image build. Porting EQL's `eql-*` tag trigger into it would have added a second dead trigger. The EQL docs rebuild is now a job in `release.yml` that fires the webhook from inside the publishing run; the `@cipherstash/stack@*` half is left alone as a pre-existing bug, recorded in that file's header so the next person does not repeat the mistake. `release-plz.yml`'s "refusing to publish the DEV placeholder" guard grepped for `eql_v3`, which appears 23,723 times in every build of that bundle including a DEV one. It now matches the schema version stamp against the crate version — the only line in the bundle that records which build produced it. `classify` accepted any version containing a hyphen where `prepare-bindings-assets.sh` requires `X.Y.Z-(alpha|beta|rc).N`. Under the loose check, `3.0.6-beta` got a public tag and GitHub release before the npm job died on it. Also: `PRE_GA_LATEST` was still `true` in the EQL publish script, so the first prerelease cut through the new job would have moved the `latest` dist-tag off the GA release. The file's own comment said to flip it once 3.0.0 GA shipped; npm's `latest` is 3.0.5. Two documentation defects of my own (a paragraph duplicated into AGENTS.md, and a SECURITY.md sentence contradicting the permissions split this PR introduces), one stale plan checkbox, an over-deep checkout, and a module-scope `execFileSync` whose failure took 40 unrelated assertions with it. Comment volume across everything this branch added is cut by roughly half. Claude-Session: https://claude.ai/code/session_01T26DhFPkLfN3qPFDq2ttKw --- .github/workflows/_build-eql-docs.yml | 30 +- .github/workflows/_build-eql-sql.yml | 69 ++--- .github/workflows/lint-release.yml | 19 +- .github/workflows/rebuild-docs.yml | 28 +- .github/workflows/release-plz.yml | 150 +++------- .../workflows/release-postgres-eql-image.yml | 83 ++---- .github/workflows/release.yml | 282 +++++++----------- AGENTS.md | 61 ++-- SECURITY.md | 23 +- .../2026-08-13-eql-monorepo-absorption.md | 2 +- packages/eql/docs/development/releasing.md | 9 +- .../eql/packages/eql/scripts/npm-publish.mjs | 13 +- scripts/__tests__/eql-pipeline-armed.test.mjs | 39 +-- scripts/__tests__/eql-suite-ci.test.mjs | 131 ++++---- .../workflow-dispatch-job-conditions.test.mjs | 50 +--- .../workflow-publish-permissions.test.mjs | 25 +- scripts/eql-pipeline-armed.mjs | 81 ++--- scripts/lint-no-workflow-caching.mjs | 87 ++---- 18 files changed, 430 insertions(+), 752 deletions(-) diff --git a/.github/workflows/_build-eql-docs.yml b/.github/workflows/_build-eql-docs.yml index ae5b5e325..bd12aa93f 100644 --- a/.github/workflows/_build-eql-docs.yml +++ b/.github/workflows/_build-eql-docs.yml @@ -1,27 +1,19 @@ name: "Build EQL docs (reusable)" -# Reusable docs build + attach for the EQL release line. Called inline by -# `release.yml` on BOTH the production and the prerelease path. The target -# release already exists by the time this runs: `_build-eql-sql.yml` creates it -# earlier in the same run, which is why both callers declare it as a `needs:`. +# Builds the EQL docs bundle and attaches it to the release _build-eql-sql.yml +# created earlier in the same run. Called by release.yml on both paths. # -# Ported from `packages/eql/.github/workflows/_build-docs.yml` — see -# `_build-eql-sql.yml`'s header for the rename, the two `packages/eql` -# rewrites, and why `defaults.run.shell: bash {0}` was dropped. +# Ported from packages/eql/.github/workflows/_build-docs.yml — see +# _build-eql-sql.yml for the rename and the path rewrites. # -# THIS PIPELINE HAS NEVER EXECUTED IN THIS REPOSITORY. `docs:generate:json`'s -# only caller was the unported copy of this file, which is why it carried a -# written exemption in `scripts/__tests__/eql-suite-ci.test.mjs` until this -# workflow landed. The doxygen half is the part to watch: `docs:generate` shells -# out to a `doxygen` binary that mise does not provide (`packages/eql/mise.toml` -# pins `python = "3.13"` under `[tools]`, and nothing else), so the apt install -# below is load-bearing and is the only thing standing between this job and a -# `doxygen: command not found` on the first real release. +# This pipeline has never executed here: `docs:generate:json` had no caller at +# all until this file landed. The apt install below is load-bearing — mise does +# not provide doxygen. on: workflow_call: inputs: ref: - description: "Git ref/SHA to build docs from. Empty -> default checkout (github.sha)." + description: "Git ref/SHA to build docs from. Empty -> default checkout." required: false type: string default: "" @@ -58,14 +50,10 @@ jobs: with: version: 2026.4.0 install: true - # Reached only from `release.yml`; see `_build-eql-sql.yml` for why a - # publish path restores no cache. Upstream had `cache: true`. - cache: false + cache: false # publish path; see _build-eql-sql.yml working_directory: packages/eql - name: Install Doxygen - # Not from mise: `packages/eql/mise.toml` does not pin doxygen, and - # `tasks/docs/generate.sh` invokes the binary directly. run: | sudo apt-get update sudo apt-get install -y doxygen diff --git a/.github/workflows/_build-eql-sql.yml b/.github/workflows/_build-eql-sql.yml index bbe6b0799..99d5f72f6 100644 --- a/.github/workflows/_build-eql-sql.yml +++ b/.github/workflows/_build-eql-sql.yml @@ -1,53 +1,22 @@ name: "Build EQL SQL (reusable)" -# Reusable SQL build + attach for the EQL release line. Called inline by -# `release.yml` on BOTH the production and the prerelease path, so an EQL SQL -# release has exactly one build path regardless of which one produced it. +# Builds the EQL SQL bundle and attaches it to a GitHub release. Called by +# release.yml on both the production and the prerelease path, so an EQL SQL +# release has one build path. # -# Ported from `packages/eql/.github/workflows/_build-sql.yml`, which GitHub -# never read — see the header of `test-eql.yml` for the same story about the -# test suite. Four things changed on the way in, each silent when wrong: +# Ported from packages/eql/.github/workflows/_build-sql.yml, which GitHub never +# read. Renamed on the way in (nothing binds to the filename; "SQL" names +# nothing on its own in a monorepo), and every path moved down one level. # -# * RENAMED. Upstream this file was `_build-sql.yml`, in a repository whose -# only artefact was SQL. Here `_build-ffi-artifacts.yml` is the sibling and -# "SQL" names nothing on its own, so the file follows the `*-eql.yml` -# convention `test-eql.yml` / `bench-eql.yml` / `macro-expand-eql.yml` -# already set. Nothing binds to the filename: it is reachable only through -# a job-level `uses:` in `release.yml`. -# * `defaults.run.working-directory: packages/eql` — `mise run build` and -# `release/*.sql` are written against the EQL root, which is one level down -# here. Without it they resolve against the monorepo root. -# * `working_directory:` on the mise step — `defaults.run` does not reach a -# `uses:` step, and mise reads config from the current directory and its -# PARENTS only, so an action running at the repo root never sees -# `packages/eql/mise.toml`. It installs nothing and leaves the config -# untrusted, and the first `mise run` fails with "Config files … are not -# trusted" — which reads as a broken toolchain rather than a wrong -# directory. -# * artifact and release `path:` / `files:` values are workspace-root -# relative and `defaults.run` does not touch them either, so they carry the -# `packages/eql/` prefix explicitly. -# -# `defaults.run.shell` is deliberately NOT overridden, where the imported file -# carried `bash {0}`. GitHub's default `bash` runs with `-eo pipefail`; `bash -# {0}` gives exactly the flags asked for, which is none of them, and a failing -# first line of a multi-command block is then discarded. See -# `scripts/__tests__/workflow-run-fail-fast.test.mjs`. -# -# THE MULTITUDES DEPLOY PING WAS DROPPED. Upstream ended this job with a -# `continue-on-error` curl to `api.developer.multitudes.co` authenticated with -# `MULTITUDES_ACCESS_TOKEN`. That secret does not exist in this repository -# (`gh secret list`), so the step would have failed on every production release -# and been swallowed by its own `continue-on-error` — indistinguishable from -# one that worked. Deploy tracking for `cipherstash/stack` is a repository-wide -# decision, not something an EQL SQL release should carry alone; if it is -# wanted, it arrives with the secret. +# The Multitudes deploy ping upstream ended this job with was dropped: +# MULTITUDES_ACCESS_TOKEN does not exist in this repository, so the step would +# have failed on every release under its own `continue-on-error`. on: workflow_call: inputs: ref: - description: "Git ref/SHA to build from. Empty -> default checkout (github.sha)." + description: "Git ref/SHA to build from. Empty -> default checkout." required: false type: string default: "" @@ -78,7 +47,9 @@ env: defaults: run: + # Every `run:` here is written against the EQL root, one level down. working-directory: packages/eql + # No `shell: bash {0}` override (upstream had one): that disables errexit. permissions: contents: write @@ -99,23 +70,21 @@ jobs: with: version: 2026.4.0 install: true - # `cache: false` IS NOT THE DEFAULT — mise-action caches unless told - # otherwise, and `scripts/lint-no-workflow-caching.mjs` forbids a - # GitHub Actions cache restore anywhere an artefact gets published. - # This workflow is reached only from `release.yml`, the job that holds - # the npm publishing credential, so a poisoned toolchain entry here - # would compile the SQL that ships. Upstream had `cache: true`. + # Defaults to true; forbidden on a publish path by + # scripts/lint-no-workflow-caching.mjs. Upstream had `cache: true`. cache: false + # `defaults.run` does not reach a `uses:` step, and mise reads config + # from cwd and its parents only. working_directory: packages/eql - name: Build EQL release - # Strip `eql-` so `eql_v3.version()` reports bare semver. An empty TAG - # intentionally falls through `tasks/build.sh`'s `${usage_version:-DEV}` - # default. + # Strip `eql-` so eql_v3.version() reports bare semver. An empty TAG + # falls through to tasks/build.sh's `${usage_version:-DEV}` default. env: TAG: ${{ inputs.tag }} run: mise run build --version "${TAG#eql-}" + # Artifact and release paths are workspace-root relative. - name: Upload EQL artifacts uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: diff --git a/.github/workflows/lint-release.yml b/.github/workflows/lint-release.yml index c4b1f4d35..0125be7de 100644 --- a/.github/workflows/lint-release.yml +++ b/.github/workflows/lint-release.yml @@ -11,13 +11,10 @@ name: Lint release tooling # avoid `sed`-into-`export` (SC2001, SC2155), `ls | wc -l` (SC2012), and # dollar-braces or backticks inside single-quoted `node -e` arguments (SC2016). # -# THE EQL RELEASE WORKFLOWS ARE IN SCOPE, and that is the whole reason their -# `actionlint` cleanliness is a property rather than a coincidence. They were -# ported from `packages/eql/.github/`, where an EQL-local `lint-release.yml` -# linted them and — like everything else in that directory — ran on nothing. -# Its two unique checks came with them: `shellcheck` over the -# `prepare-bindings-assets` wrapper and that wrapper's own bash unit test, both -# of which ran NOWHERE in this repository until this file named them. +# The EQL release workflows are in scope. They came from `packages/eql/.github/`, +# where an EQL-local copy of this file linted them and ran on nothing — along +# with its two unique checks, the `prepare-bindings-assets` shellcheck and that +# wrapper's bash unit test, neither of which ran anywhere here until now. on: pull_request: @@ -100,11 +97,9 @@ jobs: .github/workflows/release-postgres-eql-image.yml \ .github/workflows/lint-release.yml - # The EQL half. `prepare-bindings-assets.sh` is what stamps the SQL bundle - # and rewrites the four release manifests during `pnpm run version`, so a - # shell mistake in it lands mid-release with every manifest already - # rewritten. Neither of these ran anywhere in this repository before the - # release port. + # `prepare-bindings-assets.sh` stamps the SQL bundle and rewrites the four + # release manifests during `pnpm run version`, so a shell mistake in it + # lands mid-release with every manifest already rewritten. - name: shellcheck (EQL release wrappers) run: | set -euo pipefail diff --git a/.github/workflows/rebuild-docs.yml b/.github/workflows/rebuild-docs.yml index 78795b5fb..888cc3f29 100644 --- a/.github/workflows/rebuild-docs.yml +++ b/.github/workflows/rebuild-docs.yml @@ -1,27 +1,21 @@ name: Rebuild Docs -# Fires the docs-site rebuild webhook when a release tag lands. +# ⚠️ THIS WORKFLOW HAS NEVER RUN. `gh run list --workflow rebuild-docs.yml` is +# empty, across @cipherstash/stack@1.0.0, 1.1.0 and 1.1.1. A ref created with +# the repository's GITHUB_TOKEN does not start a workflow run, and +# changesets/action creates these tags with that token — so the `push` event +# below is never delivered. Valid file, configured secret, existing tags, and +# the docs site rebuilt by nothing. # -# TWO TAG FAMILIES, ONE WEBHOOK. This file used to watch only the changesets tag -# for `@cipherstash/stack`. The EQL port added the second family: the imported -# copy at `packages/eql/.github/workflows/rebuild-docs.yml` watched `eql-*` and -# ran nowhere, because GitHub reads workflows from the repository root alone. -# Merging them rather than keeping two files was the obvious call — the job is -# one `curl` against one secret, and nothing binds to either filename. -# -# `!eql-bindings-*` IS PART OF THE RELEASE IDENTITY, not a tidy-up. release-plz -# tags the Rust crate `eql-bindings-v` (its default workspace tag), and -# a crate release must not fire the docs rebuild — nor the SQL build or the -# Postgres image, which is why `packages/eql/release-plz.toml` chose a tag shape -# the `eql-*` automation does not match. GitHub applies these patterns in order: -# a ref matching a negative pattern AFTER a positive one is excluded, so -# `eql-bindings-v3.0.6` matches `eql-*` and is then dropped. +# Left as-is because fixing it means changing the JS release path, which the +# pull request that discovered it was not doing. DO NOT ADD A TAG PATTERN HERE +# expecting it to fire. The EQL half was routed around it: it is now the +# `eql-docs-rebuild` job in release.yml, firing the webhook from inside the run +# that published. The same fix applies here. on: push: tags: - '@cipherstash/stack@*' - - 'eql-*' - - '!eql-bindings-*' permissions: contents: read diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index 33822e460..511126af6 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -1,56 +1,23 @@ name: "Release eql-bindings (crates.io)" -# Publishes the `eql-bindings` crate to crates.io via release-plz (crates.io -# Trusted Publishing over OIDC — no long-lived token; GPG-signed release -# commits and tags). +# Publishes the `eql-bindings` crate via release-plz (crates.io Trusted +# Publishing over OIDC — no CARGO_REGISTRY_TOKEN). # -# THE FILENAME IS LOAD-BEARING AND MUST NOT CHANGE. crates.io Trusted -# Publishing binds a crate to a repository AND a workflow filename, exactly as -# npm's does. `_build-sql.yml` and `_build-docs.yml` could be renamed on the way -# in (see `_build-eql-sql.yml`); this one could not. +# THE FILENAME CANNOT CHANGE: crates.io binds the publisher to it. # -# PUBLISH-ONLY. Crate versioning is owned by CHANGESETS, not release-plz. The -# Version Packages PR bumps `packages/eql/crates/eql-bindings/Cargo.toml` to the -# lockstep version V (`scripts/sync-lockstep-versions.mjs`, driven by the root -# `version` script) alongside the npm package. This workflow publishes the -# COMMITTED version — a no-op when V is already on crates.io — and tags -# `eql-bindings-v`. There is deliberately no release-plz `release-pr` job: a -# release-plz version PR would fight the changesets one. +# Publish-only. Changesets owns the version — the Version Packages PR bumps +# crates/eql-bindings/Cargo.toml in lockstep with the npm package via +# scripts/sync-lockstep-versions.mjs. A release-plz `release-pr` job would +# fight it, so there isn't one. The tag is `eql-bindings-v`, chosen so a +# crate release does not fire the `eql-*` SQL/docs/image automation. # -# THE `eql-bindings-v*` TAG SHAPE IS PART OF THE RELEASE IDENTITY. It is -# release-plz's default workspace tag, and it deliberately does NOT match the -# `eql-` tags the SQL surface releases under. `rebuild-docs.yml` carries -# the matching `!eql-bindings-*` exclusion, so a crate release never fires the -# docs-site rebuild, the SQL build or the Postgres image. -# -# ## Inert until the cutover -# -# `@cipherstash/eql` is still published from `cipherstash/encrypt-query-language` -# — including this crate, whose crates.io Trusted Publishing configuration names -# that repository and this filename. So the `release` job below is gated on -# `scripts/eql-pipeline-armed.mjs`, which reads `FROZEN_PUBLISHERS` in -# `scripts/release-gate.mjs`. Deleting that entry at the Phase-5 cutover is what -# arms this workflow; there is no separate flag to remember. Read that script's -# header for why the switch is derived rather than flipped, and why the npm -# release gate could not be reused here (crates.io is not a registry it models, -# and keying on the npm answer would race `release.yml` on the very push that -# releases a version). -# -# ## Before the cutover arms this, two things must exist -# -# 1. `GPG_PRIVATE_KEY` in this repository's secrets. It does not exist today -# (`gh secret list`), and the import step below fails without it. -# 2. A crates.io Trusted Publishing entry on the `eql-bindings` crate naming -# Repository `cipherstash/stack` and Workflow `release-plz.yml`. -# -# crates.io auth is OIDC (`id-token: write`), so there is NO -# CARGO_REGISTRY_TOKEN. GitHub PR/tag operations use the default GITHUB_TOKEN. +# INERT until the Phase-5 cutover: the `release` job is gated on +# scripts/eql-pipeline-armed.mjs, which reads FROZEN_PUBLISHERS. Before arming +# it, this repository also needs a GPG_PRIVATE_KEY secret (it has none) and a +# crates.io Trusted Publishing entry naming cipherstash/stack + release-plz.yml. -# READ-ONLY BY DEFAULT; the one job that publishes escalates for itself. Same -# rule and same reason as `release.yml`: a workflow-level `id-token: write` is a -# default rather than a ceiling, so every job in a file registered as a trusted -# publisher would be able to mint a token the registry accepts. Enforced by -# `scripts/__tests__/workflow-publish-permissions.test.mjs`. +# Read-only by default; the publishing job escalates for itself. Enforced by +# scripts/__tests__/workflow-publish-permissions.test.mjs. permissions: contents: read @@ -60,18 +27,14 @@ on: - main workflow_dispatch: {} -# Static group: push-to-main and workflow_dispatch share it so two release-plz -# invocations never race a tag or a publish. Never cancel — a cancelled release -# could leave a half-published state. +# Never cancel — a cancelled release can leave a half-published state. concurrency: group: release-plz cancel-in-progress: false jobs: - # WHETHER THIS REPOSITORY MAY PUBLISH THE CRATE AT ALL. A checkout and one - # `node` call reading a Map — no registry lookup, so it cannot disagree with - # itself between runs or race the npm publish happening in `release.yml` on - # the same push. + # No registry lookup, so it cannot race release.yml's npm publish on the same + # push. See scripts/eql-pipeline-armed.mjs. eql-armed: name: Is the EQL release line armed? runs-on: ubuntu-latest @@ -86,11 +49,9 @@ jobs: - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: 22 - # No cache anywhere in a publishing workflow — see `release.yml`. package-manager-cache: false - # No pnpm, no install: the script imports node builtins and - # `scripts/release-gate.mjs`, which does the same. + # No pnpm install: the script imports node builtins only. - name: Read the publisher switch id: armed run: node scripts/eql-pipeline-armed.mjs @@ -99,23 +60,18 @@ jobs: name: "Release" needs: [eql-armed] if: needs.eql-armed.outputs.armed == 'true' - # GitHub-hosted, not Blacksmith. crates.io Trusted Publishing exchanges a - # GitHub OIDC token, and the equivalent npm path rejects self-hosted runners - # outright (E422); this repository's other publishing jobs are all - # `ubuntu-latest` for that reason and this one matches them rather than - # keeping upstream's Blacksmith label. + # GitHub-hosted, matching this repo's other publishing jobs. runs-on: ubuntu-latest timeout-minutes: 30 permissions: - contents: write # release-plz creates the eql-bindings-v tag and release - pull-requests: write # release-plz reads/links PRs when writing release notes - id-token: write # crates.io Trusted Publishing (OIDC) + contents: write # the eql-bindings-v tag and release + pull-requests: write # release-plz links PRs in release notes + id-token: write # crates.io Trusted Publishing steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - # Sign release-plz's commits and tags with the shared release key. - name: Import GPG key uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 with: @@ -124,51 +80,38 @@ jobs: git_commit_gpgsign: true git_tag_gpgsign: true - # NO mise STEP, where upstream had one. Nothing in this job runs a mise - # task: the guard below is `curl` + `jq` + `grep` from the runner image, - # and `release-plz/action` needs only `cargo` and `git`, both of which - # `ubuntu-latest` ships. Installing mise here would have cost a third-party - # action inside the job that holds the crates.io credential, and it would - # have had to carry `add_shims_to_path: false` (the rule - # scripts/__tests__/workflow-mise-setup.test.mjs applies to every - # `id-token: write` job) — at which point mise's toolchain would not be on - # PATH and the step would be installing tools nothing could reach. - # - # The pinned toolchain is not what is lost. `packages/eql/mise.toml` pins - # `rust = { version = "latest" }`, so it never named a fixed compiler; the - # image's rustup gives a recent stable and `cargo publish` packages the - # committed tree either way. + # No mise step (upstream had one): nothing here runs a mise task, and + # release-plz needs only cargo and git, which the runner image ships. + - name: Verify bundled SQL matches the crate version - # The crate bundles its SQL via `include_str!` on COMMITTED files, and - # release-plz publishes the committed tree verbatim. Refuse to publish a - # crate whose bundled SQL was not prepared for this exact version — a - # hand-pinned Cargo.toml without `pnpm run version` would otherwise ship - # the DEV placeholder as its "exact SQL". Enforced only when this run - # would actually publish (the committed version is not on crates.io - # yet): routine no-op runs on main between releases carry the DEV - # placeholder legitimately. A crates.io API failure fails towards - # ENFORCING the guard, never towards skipping it. + # The crate `include_str!`s COMMITTED SQL and release-plz publishes the + # committed tree verbatim, so a hand-pinned Cargo.toml would ship the + # DEV placeholder as its "exact SQL". Only enforced when this run would + # actually publish; a crates.io API failure fails towards enforcing. working-directory: packages/eql run: | set -euo pipefail cargo_version="$(grep -m1 '^version = ' crates/eql-bindings/Cargo.toml | cut -d'"' -f2)" - # crates.io rejects curl's default User-Agent with a 403. Without an - # explicit UA the request always fails, `|| true` swallows it, and the - # no-op short-circuit below is unreachable. + # crates.io 403s curl's default User-Agent, which would make the + # short-circuit below unreachable. published="$(curl -fsSL --retry 3 -H "User-Agent: cipherstash-eql-release (https://github.com/cipherstash/stack)" "https://crates.io/api/v1/crates/eql-bindings/versions" | jq -r '.versions[].num' | grep -Fx "$cargo_version" || true)" if [[ -n "$published" ]]; then - echo "eql-bindings@${cargo_version} is already on crates.io; this run is a publish no-op — skipping the bundled-SQL guard" + echo "eql-bindings@${cargo_version} is already on crates.io; publish no-op — skipping the bundled-SQL guard" exit 0 fi manifest_version="$(jq -r '.eqlVersion' crates/eql-bindings/sql/release-manifest.json)" if [[ "$manifest_version" != "$cargo_version" ]]; then - echo "::error::packages/eql/crates/eql-bindings/sql/release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml version ('$cargo_version') — the bundled SQL was not prepared for this release. Run 'pnpm run version' from the repository root (or 'mise run release:prepare_bindings_assets --version $cargo_version' from packages/eql) and commit the result." >&2 + echo "::error::release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml ('$cargo_version'). Run 'pnpm run version' from the repository root." >&2 exit 1 fi - # `grep -q` on a FILE, not a pipeline: there is no upstream writer to - # take SIGPIPE. See scripts/__tests__/workflow-grep-q-pipelines.test.mjs. - if ! grep -q "eql_v3" crates/eql-bindings/sql/cipherstash-encrypt.sql; then - echo "::error::packages/eql/crates/eql-bindings/sql/cipherstash-encrypt.sql looks like the DEV placeholder — refusing to publish it as release SQL." >&2 + # THE STAMP, not the schema name. Upstream grepped for `eql_v3`, which + # appears 23,000 times in every build including a DEV one, so the + # guard could not fire. tasks/build.sh writes the version into this + # COMMENT and it is the only record of which build produced the bytes + # — the manifest is regenerated alongside them and goes on agreeing. + if ! grep -q "COMMENT ON SCHEMA eql_v3 IS '${cargo_version}'" crates/eql-bindings/sql/cipherstash-encrypt.sql; then + stamped="$(grep -o "COMMENT ON SCHEMA eql_v3 IS '[^']*'" crates/eql-bindings/sql/cipherstash-encrypt.sql | head -1)" + echo "::error::bundled SQL was not built for ${cargo_version} — it carries ${stamped:-no version stamp}. Run 'pnpm run version' from the repository root." >&2 exit 1 fi @@ -176,13 +119,10 @@ jobs: uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5 with: command: release - # BOTH PATHS ARE REQUIRED HERE AND WERE ABSENT UPSTREAM, where the - # repository root WAS the Cargo workspace root. A composite action's - # steps do not inherit the caller's `defaults.run.working-directory`, - # so `working-directory:` on this step would do nothing — the paths - # have to be passed as inputs. Without them release-plz reads the - # monorepo root, finds no `[workspace]` naming `eql-bindings`, and - # publishes nothing while exiting 0. + # Required here and absent upstream: a composite action's steps do not + # inherit the caller's working-directory, so without these release-plz + # reads the monorepo root, finds no `eql-bindings` workspace, and + # exits 0 having published nothing. manifest_path: packages/eql/Cargo.toml config: packages/eql/release-plz.toml env: diff --git a/.github/workflows/release-postgres-eql-image.yml b/.github/workflows/release-postgres-eql-image.yml index 7f1c53291..c4006fd27 100644 --- a/.github/workflows/release-postgres-eql-image.yml +++ b/.github/workflows/release-postgres-eql-image.yml @@ -1,32 +1,16 @@ name: "Release Postgres + EQL image" -# Builds `ghcr.io/cipherstash/postgres-eql` — Postgres with the EQL SQL surface -# already installed — for PG 14 through 17, and pushes it to GHCR. +# Builds ghcr.io/cipherstash/postgres-eql (Postgres with EQL preinstalled) for +# PG 14-17 and pushes it. # -# Ported from `packages/eql/.github/workflows/release-postgres-eql-image.yml`. -# Every path it names moved one level down; see `_build-eql-sql.yml`'s header -# for the `defaults.run.working-directory` / `working_directory:` split and why -# `defaults.run.shell: bash {0}` was dropped. +# Dispatch-only: release.yml dispatches it on production finals. Not +# `on: release`, because a release created with GITHUB_TOKEN does not trigger +# that event — whereas a GITHUB_TOKEN workflow_dispatch does. # -# DISPATCH-ONLY, and that is deliberate rather than incidental. `release.yml` -# dispatches it on production finals with `update_floating_tags=true`. It is not -# `on: release`, because a release created by GITHUB_TOKEN does not trigger -# `on: release` fan-out — whereas a GITHUB_TOKEN `workflow_dispatch` does. -# -# ## Inert until the cutover -# -# The `eql-armed` job below reads `scripts/eql-pipeline-armed.mjs`, so a manual -# dispatch does nothing while `@cipherstash/eql` is still published from -# `cipherstash/encrypt-query-language`. `release.yml`'s dispatch is behind the -# same switch, so the two cannot disagree. Deleting the `FROZEN_PUBLISHERS` -# entry at the Phase-5 cutover arms both. -# -# The GHCR package itself is a second, independent gate that this workflow does -# not control: `ghcr.io/cipherstash/postgres-eql` is currently linked to -# `cipherstash/encrypt-query-language`, so this repository's GITHUB_TOKEN has no -# write access to it until the package's own repository access is updated. That -# has to happen with the cutover, and it is not something a workflow can assert -# ahead of time — the first armed run is what proves it. +# INERT until the cutover via the eql-armed job. A second gate is outside this +# file's control: the GHCR package is still linked to +# cipherstash/encrypt-query-language, so this repository has no write access to +# it until that moves. on: workflow_dispatch: @@ -50,8 +34,6 @@ permissions: contents: read jobs: - # See `release-plz.yml` for why the switch is derived from - # `FROZEN_PUBLISHERS` rather than being a flag somebody flips. eql-armed: name: Is the EQL release line armed? runs-on: ubuntu-latest @@ -82,11 +64,9 @@ jobs: run: working-directory: packages/eql outputs: - # Used for image tags, e.g. "3.0.6" -> ghcr.io/.../postgres-eql:17-3.0.6 eql_version: ${{ steps.ver.outputs.eql_version }} - # Passed to `mise run build --version`. Bare semver (no `eql-` prefix), - # matching the SQL release build (`_build-eql-sql.yml`), so - # `eql_v3.version()` in the image is byte-identical to the SQL release. + # Bare semver, passed verbatim to `mise run build --version`, so + # eql_v3.version() in the image matches the SQL release byte for byte. build_version: ${{ steps.ver.outputs.build_version }} update_floating_tags: ${{ steps.ver.outputs.update_floating_tags }} @@ -99,17 +79,11 @@ jobs: with: version: 2026.4.0 install: true - # This workflow pushes an image, so it is a publish path: no cache. - # Upstream had `cache: true`. Enforced by - # `scripts/lint-no-workflow-caching.mjs`. - cache: false + cache: false # pushes an image; publish path. Upstream had `true`. working_directory: packages/eql - name: Compute EQL version and tag policy id: ver - # `eql_version` arrives as bare semver (`release.yml` sends - # `needs.release.outputs.eql_version`, e.g. "3.0.6"), used both as the - # image tag and, verbatim, as the `mise run build --version` value. env: INPUT_VERSION: ${{ inputs.eql_version }} INPUT_FLOATING: ${{ inputs.update_floating_tags }} @@ -129,7 +103,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: eql-sql - # Workspace-root relative: `defaults.run` does not reach a `uses:` step. + # Workspace-root relative; `defaults.run` does not reach a `uses:` step. path: packages/eql/release/cipherstash-encrypt.sql build-images: @@ -140,7 +114,7 @@ jobs: timeout-minutes: 30 permissions: contents: read - packages: write # push to ghcr.io + packages: write strategy: fail-fast: false matrix: @@ -158,23 +132,18 @@ jobs: uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: eql-sql - # The Dockerfile `COPY`s the bundle from its own build context, which - # is `packages/eql/docker` below. + # The Dockerfile COPYs it from its own build context. path: packages/eql/docker/ - name: Set up QEMU uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 with: - # `cache-image` DEFAULTS TO TRUE and stores the binfmt image in the - # GitHub Actions cache. This job pushes a published image, so it - # restores nothing. See scripts/lint-no-workflow-caching.mjs. - cache-image: false + cache-image: false # defaults to true; GitHub Actions cache - name: Set up Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 with: - # `cache-binary` DEFAULTS TO TRUE — same rule, same reason. - cache-binary: false + cache-binary: false # defaults to true; GitHub Actions cache - name: Login to GHCR uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 @@ -185,8 +154,8 @@ jobs: - name: Compute image tags id: tags - # Always push the immutable :- tag. Only push the floating - # : tag on a release, or on a dispatch that opted in. + # Always the immutable :-; the floating : only on + # release or an opted-in dispatch. env: PG_VERSION: ${{ matrix.pg_version }} run: | @@ -202,11 +171,9 @@ jobs: - name: Build and push uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 - # NO `cache-from:` / `cache-to:`. Both are absent rather than disabled — - # this action has no boolean that turns caching off, and `type=gha` - # names the GitHub Actions cache directly. See the AUDITED_ACTIONS entry - # in scripts/lint-no-workflow-caching.mjs, which forbids the two inputs - # outright rather than asking for a `false` that does not exist. + # No cache-from/cache-to: `type=gha` is the GitHub Actions cache and + # this action has no boolean to disable caching. Forbidden outright by + # scripts/lint-no-workflow-caching.mjs. with: context: packages/eql/docker file: packages/eql/docker/Dockerfile @@ -219,13 +186,15 @@ jobs: promote-latest: name: Promote PG17 image to :latest and : + # No eql-armed condition: both its `needs` carry one, and a skipped `needs` + # skips this job. needs: [eql-armed, build-sql, build-images] if: needs.build-sql.outputs.update_floating_tags == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 timeout-minutes: 10 permissions: contents: read - packages: write # re-tag in ghcr.io + packages: write env: EQL_VERSION: ${{ needs.build-sql.outputs.eql_version }} @@ -243,7 +212,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag PG17 image as :latest and : - # buildx imagetools preserves multi-arch manifests; `docker tag` would not. + # imagetools preserves multi-arch manifests; `docker tag` would not. run: | docker buildx imagetools create \ -t "${IMAGE}:latest" \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bf8fa804..8da0530d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,30 +16,19 @@ name: Release JS # See https://docs.npmjs.com/trusted-publishers#supported-cicd-providers # Enforced by scripts/__tests__/workflow-publish-permissions.test.mjs. # -# --------------------------------------------------------------------------- -# TWO RELEASE LINES, ONE FILE, AND WHY -# --------------------------------------------------------------------------- +# THREE RELEASE LINES, ONE FILE: the JS packages (changesets), the seven FFI +# tarballs, and the EQL line ported here from packages/eql/.github/. They share +# a file because npm trusted publishing binds to a repository AND a workflow +# filename, so every npm publish in this repository has to happen here. # -# This file publishes the JS packages (changesets), the seven FFI tarballs, and -# — since the EQL release pipeline was ported here — the `@cipherstash/eql` npm -# package plus the SQL release, docs bundle and Postgres image that ship with -# it. They are in one file because npm trusted publishing binds a package to a -# repository AND A WORKFLOW FILENAME: every npm publish in this repository has -# to happen here, whatever else it is part of. See AGENTS.md, "Working on EQL". +# `workflow_dispatch` came with the EQL port (its prerelease path is dispatched +# against a release branch). A dispatch reaches every job in the file, so +# `classify` computes the mode once and the publishing jobs key on it rather +# than assuming a push to `main`. # -# `classify` is the first job for the same reason. `workflow_dispatch` was added -# with the EQL port, because the EQL prerelease path is dispatched against a -# release branch — and a dispatch reaches every job in the file, including the -# ones that publish. So the mode is computed once, up front, and the production -# jobs are keyed on it rather than assuming a push to `main`. -# -# WHAT IS INERT TODAY. `@cipherstash/eql` is still published from -# `cipherstash/encrypt-query-language`, so every EQL job below sits behind -# `eql-armed` (`scripts/eql-pipeline-armed.mjs`, derived from -# `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`) as well as behind `gate`, -# which refuses the release outright the moment a frozen package's committed -# version goes missing from npm. Deleting that map entry at the Phase-5 cutover -# arms the whole line; there is no separate flag to remember. +# The EQL jobs are INERT: they sit behind `eql-armed` +# (scripts/eql-pipeline-armed.mjs) as well as behind `gate`. See AGENTS.md, +# "Working on EQL". permissions: contents: read @@ -56,23 +45,16 @@ on: concurrency: ${{ github.workflow }}-${{ github.ref }} jobs: - # WHICH OF THE TWO RELEASE SHAPES THIS RUN IS, decided before anything else - # runs and read by every publishing job below. + # Which release shape this run is, read by every publishing job below. # - # production — a push to `main`, or a dispatch against it. Changesets owns - # the versions; the FFI and EQL branches follow whatever it - # published. - # prerelease — a dispatch against any OTHER branch whose HEAD commit - # subject is literally `chore(release):`. Cuts an EQL alpha/rc - # from a batching branch for pre-merge testing, without - # merging to `main`. - # skip — everything else, including a prerelease identity that has - # already been published. + # production — a push to `main`, or a dispatch against it. + # prerelease — a dispatch against any other branch whose HEAD subject is + # `chore(release):`. Cuts an EQL alpha/rc without merging. + # skip — everything else, including an already-published identity. # - # Safety on the prerelease path comes from three guards, not from the branch - # name: the marker commit, a version that must already be prerelease-shaped, - # and the tag short-circuit. Exactly `chore(release):` — a bare `release:` - # prefix must NOT re-trigger the publish pipeline for the pinned identity. + # Safety on the prerelease path is the marker commit, the version regex and + # the tag short-circuit — not the branch name. Exactly `chore(release):`; a + # bare `release:` must not re-trigger a publish. classify: name: Classify release intent runs-on: ubuntu-latest @@ -83,10 +65,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - # The classifier reads the HEAD commit subject. - fetch-depth: 0 - # The tag short-circuit below asks the API rather than the remote, so - # this job needs no pushable credential in its checkout. + # Depth 1 (the default): this reads `git log -1` and runs on every + # push to main. Upstream used `fetch-depth: 0`. The tag check below + # asks the API, not the remote, so no pushable credential either. persist-credentials: false - id: classify @@ -111,25 +92,20 @@ jobs: fi if [[ "$mode" == "prerelease" ]]; then - # TWO LEVELS DOWN. The EQL subtree root carries no package.json by - # design — upstream this path was `./packages/eql/package.json`, - # because there the repository root WAS the subtree root. Here that - # resolves to a directory with no manifest and the read throws. See - # AGENTS.md, "Working on EQL". + # Two levels down: the subtree root carries no package.json. + # Upstream this was `./packages/eql/package.json`. version="$(node -p "require('./packages/eql/packages/eql/package.json').version")" - if [[ "$version" != *-* ]]; then - echo "::error::a prerelease release commit must already pin a prerelease version in packages/eql/packages/eql/package.json" >&2 + # The same regex prepare-bindings-assets.sh enforces, because this + # job runs first and that script runs last. Upstream checked only + # for a hyphen, so `3.0.6-beta` classified as a prerelease and got + # a public tag and GitHub release before the npm job died on it. + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then + echo "::error::a prerelease release commit must pin an exact prerelease identity (X.Y.Z-(alpha|beta|rc).N) in packages/eql/packages/eql/package.json; found '${version}'" >&2 exit 1 fi - # Idempotency short-circuit: if this identity's npm tag already - # exists, the release was cut — a re-push of the marker, or a rerun, - # must not publish it twice. The individual steps are idempotent too; - # this just skips the whole pipeline up front. - # - # `git/matching-refs`, not `git/ref`: the latter answers a - # non-exact match with an ARRAY rather than an object, so `--jq - # .object.sha` yields nothing and the branch below reads "no tag". - # See the same note on `publish-ffi` below. + # Idempotency: if this identity's tag exists the release was cut, + # so a re-push of the marker must not publish it twice. + # `matching-refs`, not `git/ref` — see the note on `publish-ffi`. tag="eql-typescript-v${version}" at="$(gh api "repos/${REPO}/git/matching-refs/tags/${tag}" \ --jq ".[] | select(.ref == \"refs/tags/${tag}\") | .object.sha" 2>/dev/null || true)" @@ -146,15 +122,9 @@ jobs: echo "version=$version" } >> "$GITHUB_OUTPUT" - # MAY THIS REPOSITORY PUBLISH THE EQL RELEASE LINE AT ALL? - # - # A checkout and one `node` call reading a Map — no registry lookup, so it - # gives the same answer to every EQL job in this run and to `release-plz.yml` - # and `release-postgres-eql-image.yml`, which ask it separately. Read the - # header of scripts/eql-pipeline-armed.mjs for why the switch is DERIVED from - # `FROZEN_PUBLISHERS` rather than being a boolean somebody flips at the - # cutover: a forgotten flag fails silently, in the direction that publishes an - # npm package with no SQL release, no docs and no crate. + # May this repository publish the EQL line at all? A checkout and one `node` + # call reading a Map — no registry lookup, so every EQL job in this run gets + # the same answer. See scripts/eql-pipeline-armed.mjs. eql-armed: name: Is the EQL release line armed? runs-on: ubuntu-latest @@ -171,8 +141,7 @@ jobs: node-version: 22 package-manager-cache: false - # No pnpm, no install — the script imports node builtins and - # scripts/release-gate.mjs, which does the same. Same reasoning as `gate`. + # No pnpm install: the script imports node builtins only, as `gate` does. - name: Read the publisher switch id: armed run: node scripts/eql-pipeline-armed.mjs @@ -221,9 +190,8 @@ jobs: ffi-artifacts: name: Build FFI artifacts needs: [classify, gate] - # `mode == 'production'` is what `workflow_dispatch` cost. Without it, a - # dispatch against a feature branch whose committed FFI versions happen to - # be unpublished would build and publish them from that branch. + # `mode == 'production'` is what workflow_dispatch cost: without it a + # dispatch against a feature branch could publish FFI from that branch. if: needs.classify.outputs.mode == 'production' && needs.gate.outputs.ffi == 'true' uses: ./.github/workflows/_build-ffi-artifacts.yml with: @@ -397,9 +365,8 @@ jobs: id-token: write # npm OIDC trusted publishing contents: write # changesets commits and pushes the Version Packages branch pull-requests: write # …and opens/updates the PR for it - # What the EQL jobs below need to know, and nothing more. `published` alone - # is not enough: this job publishes every unpublished JS package, and an - # ordinary `@cipherstash/stack` release must not fire an EQL SQL release. + # `published` alone is not enough: this job publishes every unpublished JS + # package, and a `@cipherstash/stack` release must not fire an EQL one. outputs: eql_published: ${{ steps.eql.outputs.eql_published }} eql_version: ${{ steps.eql.outputs.eql_version }} @@ -530,18 +497,10 @@ jobs: # gh variable set STASH_POSTHOG_KEY --repo cipherstash/stack --body '' STASH_POSTHOG_KEY: ${{ vars.STASH_POSTHOG_KEY }} - # DID THIS RUN PUBLISH @cipherstash/eql, AND AT WHAT VERSION? - # - # `changesets/action` reports `publishedPackages` as a JSON array of - # `{name, version}`. Reading the version from there rather than from the - # tree is what makes the SQL release, the docs bundle and the image tag - # agree with what actually reached npm — a re-run against a tree whose - # version has already been published finds nothing here and the whole EQL - # branch skips, which is the correct answer. - # - # `publishedPackages` is `[]` when nothing published, and unset when the - # action took its VERSION branch instead (the Version Packages PR), so the - # step is guarded on `published` and defaults to an empty version. + # Read from `publishedPackages` rather than from the tree, so the SQL + # release, docs bundle and image tag agree with what reached npm. A re-run + # against an already-published version finds nothing and the EQL branch + # skips, which is correct. - name: Resolve the published EQL version id: eql if: steps.changesets.outputs.published == 'true' @@ -566,19 +525,14 @@ jobs: echo "eql_prerelease=${prerelease}" } >> "$GITHUB_OUTPUT" - # ------------------------------------------------------------------------- - # The EQL release line — production - # ------------------------------------------------------------------------- + # ---- The EQL release line: production ------------------------------------ # - # `@cipherstash/eql` ships as FIVE artefacts at one version: the npm package, - # the `eql-bindings` crate, the SQL bundle, the docs bundle and the - # `postgres-eql` image. Changesets publishes the first (above) and - # `release-plz.yml` publishes the second on the same push; the three below are + # EQL ships as five artefacts at one version. Changesets publishes the npm + # package (above) and release-plz.yml the crate on the same push; the rest are # built here, in the run that published, so they cannot drift from it. # - # ORDERING IS A `needs:` CHAIN, NOT A CONVENIENCE. `eql-docs` attaches to a - # GitHub release that `eql-sql` creates, and `eql-image` dispatches against - # the `eql-` tag that the same job's `target_commitish` produced. + # The `needs:` chain is load-bearing: `eql-docs` attaches to the release + # `eql-sql` creates, and `eql-image` dispatches against the tag it produced. eql-sql: name: Build and attach the EQL SQL release @@ -611,12 +565,40 @@ jobs: ref: ${{ github.sha }} tag: eql-${{ needs.release.outputs.eql_version }} + eql-docs-rebuild: + name: Rebuild the docs site + needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] + # Not a tag trigger, which is why this job exists. Upstream used + # `on: push: tags: ['eql-*']`, but the eql- tag is created by + # softprops/action-gh-release with GITHUB_TOKEN, and a ref created with that + # token starts no workflow run — the same rule `eql-image` works around, and + # the reason rebuild-docs.yml has never run. Firing from inside the + # publishing run removes the trigger from the picture. + if: >- + needs.classify.outputs.mode == 'production' && + needs.eql-armed.outputs.armed == 'true' && + needs.release.outputs.eql_published == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Send webhook + env: + WEBHOOK_URL: ${{ secrets.DOCS_WEBHOOK_URL }} + VERSION: ${{ needs.release.outputs.eql_version }} + run: | + set -euo pipefail + if [ -z "$WEBHOOK_URL" ]; then + echo "::error::DOCS_WEBHOOK_URL is not configured; the docs site will not rebuild for eql-${VERSION}" >&2 + exit 1 + fi + curl -X POST --fail-with-body "$WEBHOOK_URL" + echo "docs rebuild requested for eql-${VERSION}" + eql-image: name: Dispatch the Postgres + EQL image build needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] - # PRODUCTION FINALS ONLY: the floating :latest / : image tags must - # not move for a prerelease. An alpha image can still be built on demand - # through release-postgres-eql-image.yml's own workflow_dispatch. + # Production finals only: the floating :latest / : tags must not + # move for a prerelease. An alpha image is still buildable on demand. if: >- needs.classify.outputs.mode == 'production' && needs.eql-armed.outputs.armed == 'true' && @@ -628,39 +610,30 @@ jobs: actions: write # gh workflow run steps: - name: Dispatch release-postgres-eql-image.yml - # Dispatched, not an `on: release` fan-out: a release created by - # GITHUB_TOKEN does not trigger `on: release`, whereas a GITHUB_TOKEN - # workflow_dispatch does. - # - # Against the `eql-` TAG, not the branch. `eql-sql` (a `needs`) - # created that tag at the release commit, so the image is built from the - # exact released source even if `main` has advanced since the publish. + # Dispatched, not `on: release`: a GITHUB_TOKEN-created release does not + # trigger that event, but a GITHUB_TOKEN dispatch does. Against the + # eql- TAG that `eql-sql` created, so the image is built from + # the released source even if main has advanced. env: GH_TOKEN: ${{ github.token }} VERSION: ${{ needs.release.outputs.eql_version }} run: | set -euo pipefail - # This job deliberately has no checkout, so `--repo` is required: - # without it gh tries to infer the repository from a .git directory - # that does not exist. + # `--repo` is required: no checkout, so gh cannot infer it. gh workflow run release-postgres-eql-image.yml \ --repo "${GITHUB_REPOSITORY}" \ --ref "eql-${VERSION}" \ -f eql_version="$VERSION" \ -f update_floating_tags=true - # ------------------------------------------------------------------------- - # The EQL release line — prerelease - # ------------------------------------------------------------------------- + # ---- The EQL release line: prerelease ------------------------------------ # - # Dispatched against a batching branch whose HEAD is a `chore(release):` - # marker, so an alpha/rc can be cut for pre-merge testing without merging to - # `main`. Changesets is not involved: the version is already pinned in the - # package manifest, and `prerelease-eql-npm` publishes it directly. + # Dispatched against a branch whose HEAD is a `chore(release):` marker, so an + # alpha/rc can be cut without merging to `main`. Changesets is not involved — + # the version is already pinned in the manifest. # - # SQL AND DOCS BUILD FIRST, and the npm publish `needs:` both. The npm tarball - # embeds the SQL assets, so a failure to build them must not arrive after the - # package is on the registry. + # SQL and docs build first and the npm publish `needs:` both: the tarball + # embeds the SQL assets, so a build failure must not arrive after the publish. prerelease-eql-sql: name: Build and attach the prerelease EQL SQL @@ -722,14 +695,8 @@ jobs: node-version: 22 package-manager-cache: false - # `add_shims_to_path: false` and `env: false` ARE LOAD-BEARING here, for - # exactly the reasons spelled out at length on the `release` job above: - # `packages/eql/mise.toml` pins `node = "22"` under `[tools]` and mise's - # shim directory is PREPENDED to PATH, so the default `true` would shadow - # the Node installed above and hand every later `npm` call mise's bundled - # npm 10.x — which has no OIDC trusted publishing and fails with E404. - # `env: false` keeps that file's `[env]` block (DATABASE_URL and friends, - # pointed at a Postgres this job does not have) out of GITHUB_ENV. + # `add_shims_to_path: false` and `env: false` are load-bearing — see the + # `release` job above for both. - name: Install mise (release:prepare_bindings_assets runs through it) uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 with: @@ -744,12 +711,9 @@ jobs: - name: Upgrade npm for OIDC trusted publishing run: npm install -g npm@^11.5.1 - # node-pty's install hook falls back to `node-gyp rebuild` when no - # linux-x64 prebuild matches, and pnpm/action-setup v6 no longer ships - # node-gyp on PATH. Upstream's copy of this job had no such step because - # upstream had no node-pty; the install here would die with `sh: 1: - # node-gyp: not found`. Asserted by - # scripts/__tests__/workflow-node-gyp.test.mjs. + # node-pty's install hook falls back to `node-gyp rebuild` on Linux and + # pnpm/action-setup v6 does not ship node-gyp. Upstream had no node-pty, + # so no such step. Asserted by workflow-node-gyp.test.mjs. - name: Install node-gyp run: npm install -g node-gyp @@ -768,26 +732,21 @@ jobs: } - name: Prepare the exact SQL assets - # `--force` is inside the task: `mise run build --version` does not - # treat `--version` as a cache-key input, so on unchanged SQL and Rust - # it would re-serve whatever version the previous build stamped. See - # AGENTS.md, "Working on EQL". + # The task passes `--force` itself: `mise run build --version` does not + # treat `--version` as a cache-key input. See AGENTS.md. working-directory: packages/eql env: VERSION: ${{ needs.classify.outputs.version }} run: mise run release:prepare_bindings_assets --version "$VERSION" - # THROUGH TURBO, not `pnpm --filter … build`, which is what upstream ran. - # `@cipherstash/eql#build` declares `dependsOn: ["^build"]`, and a bare - # pnpm filter builds only that package — so its workspace dependencies are - # whatever an earlier step happened to leave behind. Asserted by - # scripts/__tests__/workflow-turbo-build-deps.test.mjs. + # Through turbo, not `pnpm --filter … build` as upstream ran: + # `@cipherstash/eql#build` declares `dependsOn: ["^build"]`. Asserted by + # workflow-turbo-build-deps.test.mjs. - name: Build the package run: pnpm exec turbo run build --filter @cipherstash/eql - name: Publish the package - # TWO LEVELS DOWN — the npm package, not the subtree root. Upstream this - # was `packages/eql`, where the repository root was the subtree root. + # Two levels down — the npm package, not the subtree root. working-directory: packages/eql/packages/eql env: VERSION: ${{ needs.classify.outputs.version }} @@ -806,11 +765,8 @@ jobs: REPO: ${{ github.repository }} run: | set -euo pipefail - # Created through the REST API rather than with git: the checkout - # carries `persist-credentials: false` (deliberate hardening) and a - # git-level extraheader workaround proved unreliable upstream ("could - # not read Username"). `classify` reads this same tag to decide - # whether the identity was already released. + # Through the REST API, not git: the checkout has + # persist-credentials: false. `classify` reads this same tag. tag="eql-typescript-v${VERSION}" at="$(gh api "repos/${REPO}/git/matching-refs/tags/${tag}" \ --jq ".[] | select(.ref == \"refs/tags/${tag}\") | .object.sha" 2>/dev/null || true)" @@ -835,13 +791,9 @@ jobs: contents: write # create/update the release/eql- branch ref steps: - name: Dispatch release-plz.yml at the release commit - # release-plz refuses a detached HEAD ("cannot determine current - # branch"), so dispatching against the tag does not work. To publish - # from the EXACT commit the SQL and npm artefacts shipped from — even if - # the branch has advanced since the marker push — pin a - # `release/eql-` branch at this run's commit and dispatch - # against that. It is a stable pointer per identity: force-updated on a - # retry of the same identity, never reused across identities. + # release-plz refuses a detached HEAD, so a tag dispatch does not work. + # Pin a release/eql- branch at this run's commit instead — a + # stable pointer per identity, force-updated on a retry of the same one. env: GH_TOKEN: ${{ github.token }} VERSION: ${{ needs.classify.outputs.version }} @@ -855,19 +807,12 @@ jobs: gh api -X PATCH "repos/${REPO}/git/refs/heads/${branch}" \ -f sha="$SHA" -F force=true >/dev/null fi - # `--repo` is required: this job has no checkout, and without it gh - # tries to resolve the workflow through a local git repo. + # `--repo` is required: no checkout. gh workflow run release-plz.yml --repo "$REPO" --ref "$branch" - # ------------------------------------------------------------------------- - - # WHAT THIS RUN ACTUALLY DID, in one place. - # - # Worth a job rather than leaving it to the run page because both EQL paths - # can legitimately do NOTHING: a dispatch whose prerelease identity is already - # tagged classifies as `skip`, and every job below `classify` is then skipped - # for a reason the run page renders identically to "a dependency failed". This - # says which it was. + # What this run actually did. Worth a job because both EQL paths can + # legitimately do nothing — an already-tagged identity classifies as `skip`, + # and the run page renders that identically to "a dependency failed". eql-summary: name: EQL release summary needs: @@ -875,6 +820,7 @@ jobs: - eql-armed - eql-sql - eql-docs + - eql-docs-rebuild - eql-image - prerelease-eql-sql - prerelease-eql-docs @@ -891,6 +837,7 @@ jobs: ARMED: ${{ needs.eql-armed.outputs.armed }} EQL_SQL: ${{ needs.eql-sql.result }} EQL_DOCS: ${{ needs.eql-docs.result }} + EQL_DOCS_REBUILD: ${{ needs.eql-docs-rebuild.result }} EQL_IMAGE: ${{ needs.eql-image.result }} PRE_SQL: ${{ needs.prerelease-eql-sql.result }} PRE_DOCS: ${{ needs.prerelease-eql-docs.result }} @@ -909,6 +856,7 @@ jobs: echo "| --- | --- |" echo "| eql-sql | ${EQL_SQL} |" echo "| eql-docs | ${EQL_DOCS} |" + echo "| eql-docs-rebuild | ${EQL_DOCS_REBUILD} |" echo "| eql-image | ${EQL_IMAGE} |" echo "| prerelease-eql-sql | ${PRE_SQL} |" echo "| prerelease-eql-docs | ${PRE_DOCS} |" diff --git a/AGENTS.md b/AGENTS.md index fc774a344..dc85d0a67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -293,10 +293,8 @@ monorepo, which is where the silent failures are. dead deposit: GitHub reads workflows from the repo root and nowhere else, so the eleven files that arrived with the subtree ran on nothing. `scripts/__tests__/eql-suite-ci.test.mjs` held them as a shrinking allowlist - until the release port emptied it, and now asserts the directory does not - exist and that nothing is tracked under it. **Never put a workflow under a - package** — it executes on no event, which reads exactly like a check that - passes. That same test also asserts that the three SQLx suite tasks are + until the release port emptied it, and now asserts the directory is gone. + **Never put a workflow under a package.** That same test also asserts that the three SQLx suite tasks are invoked by name from a root workflow, that every `dorny/paths-filter` path is scoped to `packages/eql/`, and that every mise task shelling out to cargo is either reachable from a root workflow or exempted with a written reason. @@ -323,39 +321,28 @@ monorepo, which is where the silent failures are. | `release-postgres-eql-image.yml` | the GHCR image, dispatched by `release.yml` on production finals | | `rebuild-docs.yml`, `lint-release.yml` | merged into the root files of the same name | - Every one of those sits behind `scripts/eql-pipeline-armed.mjs`, which reads - `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs` and answers "may this - repository publish EQL at all?". **Deleting the `@cipherstash/eql` entry at - the Phase-5 cutover is what arms the pipeline** — there is deliberately no - second flag, because a forgotten one fails silently in the direction that - publishes an npm package with no SQL release, no docs and no crate. The gate - itself is the other half: it exits non-zero the moment a frozen package's - committed version is missing from npm, which skips `release` and everything - downstream of it. Read `node scripts/release-gate.mjs` for what it is doing - right now rather than inferring it from here. - Two things the cutover also needs, neither of which a workflow can assert - ahead of time: a `GPG_PRIVATE_KEY` secret in this repository (release-plz - signs its commit and tag), and write access from this repository to the - `ghcr.io/cipherstash/postgres-eql` package, which is currently linked to - `cipherstash/encrypt-query-language`. -- **A release workflow may not restore a cache, including a Rust one, and the - two rules that say so now know about each other.** - `scripts/lint-no-workflow-caching.mjs` forbids a GitHub Actions cache restore - anywhere an artefact is published; `eql-suite-ci.test.mjs` requires a - `Swatinem/rust-cache` step on every job that compiles Rust. Four EQL release - jobs do both, so they pay a cold cargo compile — and the exemption is derived - from the linter's own target list (read out of its success output, since the - script lints on import) rather than copied, so the two cannot disagree about - which jobs those are. - That scan now follows a task's `tasks/*.sh` delegations **transitively** - (cycle-guarded, and it throws rather than truncating past - `MAX_SCRIPT_DEPTH`), and it reads the `run:` bodies of composite actions a - workflow reaches through `uses: ./…`. Both used to stop at one hop, and both - failure directions were live: a cargo helper reached only at depth 2 dropped - out of `CARGO_TASKS` entirely — no orphan reported, no exemption demanded, - and the job running it stopped counting as a Rust job for the cache check — - while a mise task invoked from a composite action read as run by nobody, - whose natural repair is an exemption claiming CI does not run it. + Every one sits behind `scripts/eql-pipeline-armed.mjs`, which reads + `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`. **Deleting the + `@cipherstash/eql` entry at the Phase-5 cutover is what arms the pipeline** — + there is no second flag, because a forgotten one fails silently in the + direction that publishes an npm package with no SQL release, docs or crate. + Run `node scripts/release-gate.mjs` for what the gate is doing right now + rather than inferring it from here. + + **The cutover needs three more things, none of which a workflow can assert + ahead of time:** a `GPG_PRIVATE_KEY` secret (release-plz signs its commit and + tag, and this repository has none); write access from here to the + `ghcr.io/cipherstash/postgres-eql` package, still linked to + `cipherstash/encrypt-query-language`; and a docs-site rebuild that actually + fires — `rebuild-docs.yml` has never run in its life, because a ref created + with `GITHUB_TOKEN` starts no workflow. The EQL half is routed around it as a + job in `release.yml`; the `@cipherstash/stack@*` half is still broken. +- **A release workflow may not restore a cache, including a Rust one.** + `lint-no-workflow-caching.mjs` forbids a cache restore anywhere an artefact is + published; `eql-suite-ci.test.mjs` wants `Swatinem/rust-cache` on every job + that compiles Rust. Four EQL release jobs do both, so they pay a cold compile. + The exemption is derived from the linter's own target list rather than copied, + so the two cannot disagree about which jobs those are. - **The EQL path filters are three copies of one list, and the list is derived now, not remembered.** `test-eql.yml` writes it twice (an `on: push: paths:` filter deciding whether the workflow starts at all, and a diff --git a/SECURITY.md b/SECURITY.md index 13a7833a5..6896ad615 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -156,12 +156,13 @@ Both bind to a *workflow filename* at the registry, so renaming either file silently invalidates its publisher configuration. `scripts/__tests__/workflow-publish-permissions.test.mjs` holds the shape those -two files must keep: `id-token: write` is granted per job and never at workflow -level (where it would be a default inherited by every job in a file the registry -already trusts), and a job in such a file that does not publish may not hold a -writable scope. Both lists are equalities, so a new publisher — or a new job -that can write to the repository the publishers build from — has to be argued -for in the same diff. +two files must keep, as two separate equalities: who may publish (`id-token: +write`, granted per job and never at workflow level, where it would be inherited +by every job in a file the registry already trusts), and who may write to the +repository at all. They are separate because a publishing workflow also contains +jobs that create a release or dispatch another workflow — holding one does not +confer the other. Both are equalities, so either addition has to be argued for +in the same diff. [GitHub Actions cache poisoning is a known attack][1] against credential-bearing workflows. The mechanism is: @@ -179,12 +180,10 @@ We mitigate this by: `_build-eql-sql.yml`, `_build-eql-docs.yml`), `release-plz.yml` and `release-postgres-eql-image.yml` - Automated checks for disabled caching on high-risk workflows - (`scripts/lint-no-workflow-caching.mjs`). It works from an ALLOWLIST of - audited actions rather than a denylist of cache actions, so an action it has - never seen is a finding by default — the cases that matter most are - `setup-` actions that cache by default, with no `cache:` input and no - telling name. The cost is real and accepted: four EQL release jobs compile - Rust with no `Swatinem/rust-cache` restore. + (`scripts/lint-no-workflow-caching.mjs`), working from an allowlist of audited + actions rather than a denylist of cache actions — so an action it has never + seen is a finding by default. The cost is accepted: four EQL release jobs + compile Rust with no cache restore. [1]: https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/ diff --git a/docs/plans/2026-08-13-eql-monorepo-absorption.md b/docs/plans/2026-08-13-eql-monorepo-absorption.md index c626e201b..efa8d9124 100644 --- a/docs/plans/2026-08-13-eql-monorepo-absorption.md +++ b/docs/plans/2026-08-13-eql-monorepo-absorption.md @@ -323,7 +323,7 @@ Five surfaces, each flipped independently, each inert until its own cutover. - [ ] Repoint all EQL manifests at `cipherstash/stack`: `packages/eql/packages/eql/package.json` — `repository.url`, `repository.directory` → `packages/eql/packages/eql`, **and `bugs.url`** (line 13, still `encrypt-query-language/issues`, which would send users to an archived repo) — plus `crates/eql-bindings/Cargo.toml` (`repository`, `homepage`) and `release-plz.toml`'s prose. npm and crates.io both reject a publish whose repository field does not match; `bugs.url` is not enforced, which is why it is the one that survives a cutover unnoticed. - [x] Port `release.yml`'s EQL half into this repo's `release.yml` — the existing gate-then-publish structure already models "one version authority plus N idempotent publishers", which is the shape it was derived from. Preserve the `classify` prerelease path and the `eql-typescript-v` idempotency short-circuit. **What it cost, all of it found by this repo's guards rather than by review.** `workflow_dispatch` on `release.yml` (the prerelease path is dispatched) means every job in the file is reachable by hand, so `classify` gates the FFI and changesets jobs too. Three guards had to grow: `workflow-publish-permissions.test.mjs` needed `REPO_WRITE_JOBS` beside `OIDC_JOBS`, because six new jobs need a writable scope and must NOT be able to publish; `workflow-dispatch-job-conditions.test.mjs` needed a reasoned skip list, because the production and prerelease halves are mutually exclusive by construction and no single synthetic context can run both; and `eql-suite-ci.test.mjs`'s rust-cache requirement needed a publish-path exemption derived from the caching linter's own targets. Two real defects came out of the same pass: the prerelease job needed `Install node-gyp` before `pnpm install` (node-pty, which upstream did not have) and `pnpm exec turbo run build` rather than `pnpm --filter … build` (`^build` dependencies, which upstream did not have either). - [x] Port `release-plz.yml` unchanged in filename — crates.io trusted publishing binds to it. It also needed `manifest_path: packages/eql/Cargo.toml` and `config: packages/eql/release-plz.toml`, which upstream did not: a composite action's steps do not inherit the caller's `defaults.run.working-directory`, so a `working-directory:` on the step would have done nothing and release-plz would have read the monorepo root, found no workspace naming `eql-bindings`, and exited 0 having published nothing. -- [ ] Port `release-postgres-eql-image.yml` and `rebuild-docs.yml`. +- [x] Port `release-postgres-eql-image.yml` and `rebuild-docs.yml`. The image workflow kept its filename; `rebuild-docs.yml` did not survive contact. Porting its `eql-*` push-tag trigger into the root file of the same name would have added a second dead trigger to a workflow that has **never run in its life** — zero runs against `@cipherstash/stack@1.0.0`, `1.1.0` and `1.1.1`, because changesets creates those tags with `GITHUB_TOKEN` and a `GITHUB_TOKEN`-created ref does not start a `push` workflow. That is the same rule this plan already relies on for the image build. So the EQL docs-site rebuild is a job in `release.yml` that fires the webhook directly, and the `@cipherstash/stack@*` half is left as it was — a live pre-existing bug, now recorded in that file's header rather than inherited silently. - [ ] **Repoint trusted publishing**, only after a versioned dry run: npm for `@cipherstash/eql` (workflow `release.yml`, explicitly selecting `npm publish` under Allowed actions), and crates.io for `eql-bindings` (workflow `release-plz.yml`). - [ ] Move the `DOCS_WEBHOOK_URL` secret and the GHCR package ownership. - [ ] Activate the parked changesets, merge the Version Packages PR, and verify at one version V: the npm tarball, the crate, the `eql-` release with both SQL assets, the docs rebuild, and the image tags. diff --git a/packages/eql/docs/development/releasing.md b/packages/eql/docs/development/releasing.md index 643344d3e..75362279e 100644 --- a/packages/eql/docs/development/releasing.md +++ b/packages/eql/docs/development/releasing.md @@ -169,11 +169,10 @@ Three independent git tag families, all keyed to the same identity `V`: - **`eql-bindings-vV`** — the **`eql-bindings` Rust crate** on crates.io. - **`eql-typescript-vV`** — the **`@cipherstash/eql` npm package**. -The npm dist-tag is `latest` for finals — and, **until 3.0.0 final ships**, -also for prereleases (the alphas are the only release line, so `latest` -tracks the newest alpha; see `PRE_GA_LATEST` in -`packages/eql/scripts/npm-publish.mjs`). After GA, prereleases return to -their channel dist-tag (`alpha` / `beta` / `rc`). Each language package bundles the **exact** self-contained +The npm dist-tag is `latest` for finals and the channel tag (`alpha` / `beta` / +`rc`) for prereleases. Before 3.0.0 GA prereleases also took `latest`, because +the alphas were the only release line; GA has shipped, so `PRE_GA_LATEST` in +`scripts/npm-publish.mjs` is now `false`. Each language package bundles the **exact** self-contained SQL it was generated against (`eql_bindings::sql`; npm `./sql` subpath), so a consumer pins wire types and the matching DDL together. diff --git a/packages/eql/packages/eql/scripts/npm-publish.mjs b/packages/eql/packages/eql/scripts/npm-publish.mjs index ad698e10c..72a12766a 100644 --- a/packages/eql/packages/eql/scripts/npm-publish.mjs +++ b/packages/eql/packages/eql/scripts/npm-publish.mjs @@ -8,13 +8,12 @@ const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const pkg = JSON.parse( readFileSync(resolve(packageRoot, 'package.json'), 'utf8'), ) -// Dist-tag policy (2026-07-08): until 3.0.0 final ships, `latest` tracks the -// newest release INCLUDING prereleases — the 3.0.0 alphas are the only release -// line, so a bare `npm install @cipherstash/eql` should resolve to the newest -// alpha rather than a stale one. Once 3.0.0 GA is published, flip -// PRE_GA_LATEST to false so prereleases go back to their channel dist-tag -// (alpha/beta/rc) and `latest` stays on finals. -const PRE_GA_LATEST = true +// Dist-tag policy. Before 3.0.0 GA this was `true` so `latest` tracked the +// alphas. GA shipped (npm `latest` is 3.0.5), so prereleases now go to their +// channel tag and `latest` stays on finals — otherwise the next alpha would +// move `latest` off the GA release and `npm install @cipherstash/eql` would +// resolve to it. +const PRE_GA_LATEST = false const prerelease = pkg.version.match(/-(alpha|beta|rc)\./) const tag = prerelease && !PRE_GA_LATEST ? prerelease[1] : 'latest' diff --git a/scripts/__tests__/eql-pipeline-armed.test.mjs b/scripts/__tests__/eql-pipeline-armed.test.mjs index 90af29fe5..5657091ed 100644 --- a/scripts/__tests__/eql-pipeline-armed.test.mjs +++ b/scripts/__tests__/eql-pipeline-armed.test.mjs @@ -13,37 +13,28 @@ import { REPO_ROOT } from './lib/repo-root.mjs' import { readWorkflow } from './lib/workflows.mjs' /** - * The switch that keeps the EQL release pipeline inert, and the wiring that - * makes it mean anything. + * The arming switch, and the wiring that makes it mean anything. * - * The whole point of deriving the switch from `FROZEN_PUBLISHERS` is that the - * Phase-5 cutover arms the pipeline by deleting a map entry it has to delete - * anyway — the release gate refuses every release until it does. What that - * buys is undone completely by a job that stops reading the answer, and a - * dropped `if:` is invisible: while the pipeline is inert for OTHER reasons - * (the gate blocks, and `changeset publish` finds nothing), a job with no guard - * behaves exactly like a job with one. It would first differ on the day of the - * cutover, in the direction that publishes. - * - * So the wiring is asserted as an equality here, not just the script's logic. + * Deriving the switch from `FROZEN_PUBLISHERS` buys nothing if a job stops + * reading it, and a dropped `if:` is invisible: the pipeline is inert for other + * reasons too, so an unguarded job behaves exactly like a guarded one right up + * to the cutover, when it publishes. Hence the equality on the wiring below. */ const SCRIPT = join(REPO_ROOT, 'scripts/eql-pipeline-armed.mjs') /** - * Every job that must be gated on the switch, as ` / `. + * Every job that must be gated on the switch. * - * `release-postgres-eql-image.yml / promote-latest` is deliberately absent: it - * `needs:` two jobs that ARE gated, so it cannot run without them, and its own - * `if:` carries the floating-tag policy instead. Adding a second condition - * there would read as belt-and-braces and would in fact be a second place to - * forget. + * `promote-latest` is absent on purpose: it `needs:` two gated jobs, so it + * cannot run without them, and its own `if:` carries the floating-tag policy. */ const GATED_JOBS = [ '.github/workflows/release-plz.yml / release', '.github/workflows/release-postgres-eql-image.yml / build-images', '.github/workflows/release-postgres-eql-image.yml / build-sql', '.github/workflows/release.yml / eql-docs', + '.github/workflows/release.yml / eql-docs-rebuild', '.github/workflows/release.yml / eql-image', '.github/workflows/release.yml / eql-sql', '.github/workflows/release.yml / prerelease-eql-crate', @@ -82,9 +73,8 @@ describe('the switch answers from FROZEN_PUBLISHERS', () => { }) it('is armed once the entry is gone — the cutover, exercised now', () => { - // The state that does not exist yet. Without this the "armed" branch would - // first execute during the cutover, which is the worst possible moment to - // discover the switch was inverted. + // Without this the armed branch first executes at the cutover, which is the + // worst moment to discover the switch was inverted. expect(eqlPipelineArmed(new Map())).toBe(true) expect(eqlPipelineArmed(new Map([['@cipherstash/other', 'x']]))).toBe(true) }) @@ -95,8 +85,8 @@ describe('the switch answers from FROZEN_PUBLISHERS', () => { }) it('matches the live map, whichever state that is in', () => { - // Not "is currently false". The map is what Phase 5 edits, and pinning the - // verdict here would make the cutover fail in this file for no reason. + // Not "is currently false" — pinning the verdict would make the cutover + // fail here for no reason. expect(eqlPipelineArmed()).toBe(!FROZEN_PUBLISHERS.has(EQL_PACKAGE)) }) }) @@ -124,8 +114,7 @@ describe('the switch is readable by a workflow', () => { }) it('does not write GITHUB_OUTPUT when there is none', () => { - // A `main()` that assumed the variable would throw on a local run, which is - // how someone checks the answer before a cutover. + // A local run is how someone checks the answer before a cutover. const stdout = run({ GITHUB_OUTPUT: '' }) expect(stdout).toContain(EQL_PACKAGE) }) diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index c0388db8f..593f96560 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -865,12 +865,10 @@ const CI_EXEMPT_CARGO_TASKS = new Map([ 'test:lint', '`cargo fmt --check` scoped to tests/sqlx. `test:crates` runs `cargo fmt --check` at the EQL workspace ROOT, and tests/sqlx is a workspace member, so the CI run covers exactly these files and four crates more — verified against `cargo fmt --check -v`, which lists packages/eql/tests/sqlx/** among its targets. test-eql.yml says the same thing at the `rust-crates` job, which is where the standalone lint step used to be.', ], - // `docs:generate:json` WAS HERE, as category (d) — "unreached only because the - // workflow that runs it has not been ported yet" — naming - // `packages/eql/.github/workflows/_build-docs.yml` as its sole caller. That - // file is now `.github/workflows/_build-eql-docs.yml` and does run, so the - // exemption went stale exactly as it said it would, and the staleness check - // below is what forced its removal rather than a memory of having written it. + // `docs:generate:json` was here as category (d), naming the unported + // `_build-docs.yml` as its sole caller. That file now runs as + // `_build-eql-docs.yml`, so the exemption went stale exactly as it predicted + // and the staleness check below is what forced its removal. ]) /** Cargo tasks a given reachability set leaves unaccounted for. */ @@ -1294,12 +1292,9 @@ describe('every cargo check EQL owns is reached by a root workflow', () => { * actually fails. */ const SOLE_CALLER_WORKFLOWS = [ - // docs:generate:json. It arrived on this list the day the release port - // landed — until then that task was reachable from nothing at all and carried - // a written exemption saying so. Note what "sole caller" costs here that it - // does not cost for the three below: this workflow runs only on a RELEASE, so - // a break in the doxygen -> JSON pipeline surfaces mid-publish rather than on - // the pull request that caused it. + // docs:generate:json, reachable from nothing at all before the release port. + // Note what "sole caller" costs here and not for the three below: this runs + // only on a RELEASE, so a break surfaces mid-publish. `${WORKFLOW_DIR}/_build-eql-docs.yml`, `${WORKFLOW_DIR}/bench-eql.yml`, // test:bench `${WORKFLOW_DIR}/macro-expand-eql.yml`, // test:matrix:expand @@ -1361,21 +1356,12 @@ describe('the guard fails when a workflow stops calling a cargo check', () => { /** * The deposited `.github` is GONE, and must stay gone. * - * This used to be a SHRINKING ALLOWLIST — eleven entries, asserted by equality, - * emptied one commit at a time as each file was ported or deliberately dropped. - * The last of them left with the release port, which is what the allowlist's own - * failure message asked for: "When this list empties, delete the directory and - * replace this check with `expect(existsSync(...)).toBe(false)`." + * It was a shrinking allowlist of eleven files, emptied by the release port. + * The rule does not expire with it: a workflow under a package executes on no + * event, which looks exactly like a check that passes. * - * The rule it enforced does not expire with it. A workflow file under a package - * reads as live CI and is not: GitHub reads workflows from the repository root - * alone, so anything deposited here executes on no event — which is - * indistinguishable, on every run and every PR page, from a check that passes. - * That is the failure this whole file exists to prevent, and `lintWiring.test.ts` - * asserts exactly the same thing for protect-ffi's deposit. - * - * WHERE EACH FILE WENT, because "it was deleted" is not the same claim as "it - * was ported" and a reader a year from now cannot tell them apart from a diff: + * Where each file went — "deleted" and "ported" are different claims and a diff + * cannot tell them apart: * * _build-sql.yml -> .github/workflows/_build-eql-sql.yml * _build-docs.yml -> .github/workflows/_build-eql-docs.yml @@ -1486,46 +1472,48 @@ function reachesCargo(name, seen = new Set()) { /** * The workflows that MUST NOT restore a cache, read from the gate that says so. * - * THE TWO RULES IN THIS REPOSITORY POINT OPPOSITE WAYS, and the EQL release port - * is where they met. `scripts/lint-no-workflow-caching.mjs` forbids a GitHub - * Actions cache restore anywhere an artefact gets published — a poisoned entry - * would execute in the job holding the npm or crates.io credential — and - * `Swatinem/rust-cache` is precisely such a restore. The check below asks for - * one on every job that compiles Rust. Four release jobs do both: they compile - * the EQL workspace (`mise run build` shells out to `cargo run -p eql-codegen`) - * inside a workflow that publishes. - * - * Supply chain wins, so those jobs pay a cold compile. What must not happen is - * this file and that linter disagreeing about WHICH jobs those are: a workflow - * added to one list and not the other would either be told to add a cache the - * linter then rejects, or be quietly dropped from both checks. + * `lint-no-workflow-caching.mjs` forbids a cache restore in a publishing + * workflow; the check below wants a `Swatinem/rust-cache` step on every job + * that compiles Rust. Four EQL release jobs do both, so they pay a cold + * compile. Derived from the linter's own target list rather than copied, so the + * two cannot disagree about which jobs those are. * - * So the list is derived rather than copied. The linter prints its default - * TARGETS on a clean run — the same trick `lint-no-workflow-caching.test.mjs` - * uses to read them — because the script lints on import and cannot be imported - * for the constant alone. If it ever exits non-zero this throws, rather than - * falling back to an empty set: an exemption list that silently empties would - * re-report all four jobs, and one that silently fills would excuse everything. + * Lazy: the linter lints on import, so its targets are read by running it, and + * a real caching violation exits non-zero. Computing this at module scope made + * that a collect failure, taking all 40-odd assertions in this file with it. */ -const NO_CACHE_WORKFLOWS = (() => { - const stdout = execFileSync( - 'node', - [join(REPO_ROOT, 'scripts/lint-no-workflow-caching.mjs')], - { cwd: REPO_ROOT, encoding: 'utf8' }, - ) - const targets = stdout - .split('\n') - .map((line) => line.trim()) - .filter((line) => line.startsWith(`${WORKFLOW_DIR}/`)) - if (targets.length === 0) { - throw new Error( - 'lint-no-workflow-caching.mjs printed no targets. Its success output is the ' + - 'only readable copy of its TARGETS, and without it the publish-path ' + - 'exemption below silently covers nothing.', - ) +let noCacheWorkflows +function inNoCacheWorkflow(relPath) { + if (noCacheWorkflows === undefined) { + let stdout + try { + stdout = execFileSync( + 'node', + [join(REPO_ROOT, 'scripts/lint-no-workflow-caching.mjs')], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ) + } catch (err) { + throw new Error( + 'lint-no-workflow-caching.mjs exited non-zero, so its target list could ' + + 'not be read. Fix the caching finding it reports first:\n' + + String(err?.stdout ?? '') + + String(err?.stderr ?? ''), + ) + } + const targets = stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith(`${WORKFLOW_DIR}/`)) + if (targets.length === 0) { + throw new Error( + 'lint-no-workflow-caching.mjs printed no targets, so the publish-path ' + + 'exemption below would silently cover nothing.', + ) + } + noCacheWorkflows = new Set(targets) } - return new Set(targets) -})() + return noCacheWorkflows.has(relPath) +} /** * Every job in a root workflow, with the cargo-reaching mise tasks its steps @@ -1584,13 +1572,10 @@ describe('every job that compiles Rust restores the shared cache', () => { }) it('exempts the publish paths, and only those', () => { - // The exemption is worth an assertion of its own rather than only ever - // appearing as an absence in the check below: if the derivation above stops - // selecting the release jobs, they rejoin that check and the natural repair - // is to add a rust-cache step the caching gate then rejects. - const exempt = RUST_JOBS.filter(({ relPath }) => - NO_CACHE_WORKFLOWS.has(relPath), - ) + // Worth its own assertion: if the derivation stops selecting these jobs + // they rejoin the check below, whose natural repair is a rust-cache step + // the caching gate then rejects. + const exempt = RUST_JOBS.filter(({ relPath }) => inNoCacheWorkflow(relPath)) expect( exempt.map(({ relPath, jobName }) => `${relPath} (${jobName})`).sort(), 'No cargo-compiling job sits in a no-cache publish workflow any more. If the EQL release pipeline moved, move this with it; if it was deleted, delete this.', @@ -1601,8 +1586,8 @@ describe('every job that compiles Rust restores the shared cache', () => { '.github/workflows/release.yml (prerelease-eql-npm)', ]) - // …and none of them smuggled the cache in anyway. The caching gate would - // catch it too; this says so where the reader is already looking. + // …and none smuggled one in anyway. The caching gate would catch it too; + // this says so where the reader is already looking. expect( exempt.filter(({ cached }) => cached).map(({ relPath }) => relPath), 'A publish-path job carries a `Swatinem/rust-cache` step. That is a GitHub Actions cache restore inside a job holding a publishing credential — see scripts/lint-no-workflow-caching.mjs.', @@ -1611,7 +1596,7 @@ describe('every job that compiles Rust restores the shared cache', () => { it('leaves no cargo-compiling job uncached', () => { const uncached = RUST_JOBS.filter( - ({ relPath, cached }) => !cached && !NO_CACHE_WORKFLOWS.has(relPath), + ({ relPath, cached }) => !cached && !inNoCacheWorkflow(relPath), ).map( ({ relPath, jobName, compiling }) => `${relPath} (${jobName}): runs ${compiling.join(', ')}`, diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index 2d3b4f680..4611d7d5c 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -77,28 +77,16 @@ const EXPECTED_DISPATCHABLE = [ /** * Jobs that legitimately do NOT run on a plain manual dispatch, with the reason. * - * WHY THIS EXISTS AT ALL. The check below reads "a declared `workflow_dispatch` - * must actually dispatch", and it enforced that by requiring EVERY job-level - * `if:` in such a workflow to be true under one synthetic dispatch context. - * That is exact for a workflow with one path through it, which every - * dispatchable workflow here had. `release.yml` has two, and they are mutually - * exclusive by construction: `classify` reports `production` for `main` and - * `prerelease` for a marker commit on any other branch, and each half of the - * file is keyed on one of them. Under any single context one half is false. No - * spelling of the conditions avoids that, because the exclusivity is the - * design. + * The check below required EVERY job-level `if:` to be true under one synthetic + * dispatch context. Exact for a workflow with one path; `release.yml` has two, + * mutually exclusive by construction (`classify` reports `production` for main + * and `prerelease` for a marker commit elsewhere), so under any single context + * one half is false. * - * WHAT IS NOT WEAKENED. The alternative was to relax the rule to "at least one - * job runs", which would still have caught the original defect (a - * single-job workflow whose only job skipped) and would have said nothing about - * a ten-job workflow where nine skip. This keeps the per-job requirement and - * makes the exceptions an equality instead — so an unlisted job that stops - * running on dispatch fails, AND an entry that stops being needed fails. - * - * The synthetic context dispatches against `refs/heads/main`, so "production" - * is the path being modelled. That is the right default: it is what a dispatch - * of `release.yml` does unless someone deliberately points it at a release - * branch. + * Relaxing to "at least one job runs" would say nothing about a ten-job + * workflow where nine skip. The per-job requirement stays; the exceptions + * become an equality, failing in both directions. The synthetic context + * dispatches against `refs/heads/main`, so production is what is modelled. */ const DISPATCH_SKIPPED_JOBS = [ // The four prerelease jobs. A dispatch against `main` classifies as @@ -416,12 +404,9 @@ function runsWhen(condition, context) { */ const PERMISSIVE_NEEDS = { changes: { outputs: { relevant: 'true' } }, - // release.yml, release-plz.yml and release-postgres-eql-image.yml. Every one - // of these is a gate that has nothing to do with how the run was triggered, - // so it is held open — otherwise "this job skips because there is nothing to - // publish" would be indistinguishable from "this job skips on a dispatch", - // and the check below would pass for the wrong reason on the FFI jobs it has - // been covering since before the EQL port. + // The release workflows' gates, none of which is about how the run was + // triggered. Held open, or "skips because there is nothing to publish" would + // be indistinguishable from "skips on a dispatch". classify: { outputs: { mode: 'production', version: '3.0.6' } }, 'eql-armed': { outputs: { armed: 'true' } }, gate: { result: 'success', outputs: { ffi: 'true' } }, @@ -431,9 +416,8 @@ const PERMISSIVE_NEEDS = { outputs: { eql_published: 'true', eql_version: '3.0.6', - // A FINAL, not a prerelease: `eql-image` runs only for a final, and this - // is the value that lets it through. The prerelease half of the file is - // in DISPATCH_SKIPPED_JOBS rather than being served by a second context. + // A final: `eql-image` runs only for one. The prerelease half of the + // file is in DISPATCH_SKIPPED_JOBS instead. eql_prerelease: 'false', }, }, @@ -580,10 +564,8 @@ describe('a declared workflow_dispatch actually dispatches', () => { }) it('skips exactly the jobs declared unreachable by a plain dispatch', () => { - // An equality, both directions. A job that quietly stops running on - // dispatch fails; an entry left behind after its job starts running (or - // after the job is deleted) fails too, so the list cannot become a place - // findings go to be forgotten. + // Both directions, so the list cannot become a place findings go to be + // forgotten. const skipped = DISPATCHABLE_CONDITIONS.filter( ({ condition }) => !runsWhen(condition, CONTEXTS.workflow_dispatch), ).map((entry) => entry.id) diff --git a/scripts/__tests__/workflow-publish-permissions.test.mjs b/scripts/__tests__/workflow-publish-permissions.test.mjs index a7fff8be0..119ff296d 100644 --- a/scripts/__tests__/workflow-publish-permissions.test.mjs +++ b/scripts/__tests__/workflow-publish-permissions.test.mjs @@ -57,26 +57,15 @@ const OIDC_JOBS = [ ] /** - * The jobs in a publishing workflow that may hold ANY writable scope, as - * ` / `. A superset of `OIDC_JOBS`, asserted as such below. + * The jobs in a publishing workflow that may hold ANY writable scope. A + * superset of `OIDC_JOBS`, asserted as such below. * - * WHY THIS IS A SECOND LIST AND NOT THE SAME ONE. The third check below reads - * "a job in a publishing workflow that does not publish must not be able to - * write to the repository", and it enforced that by asking whether the job was - * in `OIDC_JOBS` — which made the npm-credential list do double duty as the - * write list. That was exactly right while the only writing jobs were the two - * that publish. It stopped being expressible the moment `release.yml` grew the - * EQL release line: `eql-sql` creates the `eql-` GitHub release, - * `eql-image` dispatches another workflow, `prerelease-eql-crate` moves a - * branch ref. All three need a writable scope; none of them may mint a publish - * token, and putting them in `OIDC_JOBS` to buy the write would have said they + * A second list because `OIDC_JOBS` was doing double duty as the write list — + * fine while the only writing jobs were the two that publish, and not + * expressible once `release.yml` grew jobs that create a release, dispatch a + * workflow or move a branch ref. Those need a writable scope and must not be + * able to mint a publish token; adding them to `OIDC_JOBS` would have said they * could. - * - * So the two facts are separated. `OIDC_JOBS` still answers "who may publish?" - * as an equality, which is the strong claim; this answers the weaker one, and - * every entry names the scope and what it is for — because `contents: write` in - * a file registered as a trusted publisher is still the scope that lets a - * compromised step rewrite the tree the publish jobs build from. */ const REPO_WRITE_JOBS = [ ...OIDC_JOBS, diff --git a/scripts/eql-pipeline-armed.mjs b/scripts/eql-pipeline-armed.mjs index a00d36eff..9ae50728a 100644 --- a/scripts/eql-pipeline-armed.mjs +++ b/scripts/eql-pipeline-armed.mjs @@ -1,75 +1,42 @@ /** - * Is the EQL release pipeline allowed to publish anything from THIS repository? - * - * ## Why this is a script and not a workflow input - * - * The whole EQL release pipeline — `release.yml`'s EQL jobs, `release-plz.yml`, - * `_build-eql-sql.yml`, `_build-eql-docs.yml`, - * `release-postgres-eql-image.yml` — was built here while `@cipherstash/eql` is - * still published from `cipherstash/encrypt-query-language`. It exists so it - * can be reviewed, dry-run and corrected, and it must reach no registry until - * the Phase-5 cutover repoints trusted publishing. That is a state, and a state - * needs a switch. - * - * The obvious switch is a boolean somebody flips. It is the wrong one, for the - * reason `scripts/release-gate.mjs` records about its own map: an entry left - * behind does not fail on the day it goes wrong, it fails on the next release. - * A hand-flipped flag has the same shape in reverse — the cutover PR repoints - * npm and crates.io, and NOTHING fails if it forgets the flag. The pipeline - * simply stays inert, and the first EQL release after the cutover quietly - * publishes an npm package with no SQL release, no docs and no crate. - * - * So the switch is DERIVED from the fact it depends on. `FROZEN_PUBLISHERS` in - * `scripts/release-gate.mjs` is the single written record of "this package - * lives here and is published elsewhere", and the cutover deletes its entry - * because the release gate blocks every release until it does. Deleting it is - * what arms this pipeline. There is no second edit to forget. - * - * ## Why not just use the release gate - * - * `release-gate.mjs` answers a REGISTRY question — which committed versions are - * missing from npm — and it already blocks a frozen package's release by - * exiting non-zero. That covers `release.yml`, whose EQL jobs all sit behind - * `needs: [gate]`. - * - * It does not cover `release-plz.yml`. That workflow publishes a CRATE, on its - * own `push: main` trigger, and crates.io is not a registry the gate models at - * all. Keying it on the npm answer would also be a race: on the push that - * releases version V, `release.yml` and `release-plz.yml` start together, and - * whichever npm publish finishes first would flip the other's answer from - * "missing" to "published" mid-run — so the crate would be skipped on exactly - * the push that was supposed to release it. - * - * This asks a question with no registry in it and no clock in it: is - * `@cipherstash/eql` still recorded as published from somewhere else? Same - * answer for every job in every workflow on every run. + * May this repository publish the EQL release line? + * + * The pipeline was built here while `@cipherstash/eql` is still published from + * `cipherstash/encrypt-query-language`, so it must reach no registry until the + * Phase-5 cutover. That is a state, and it needs a switch. + * + * The switch is DERIVED, not flipped. `FROZEN_PUBLISHERS` in + * `scripts/release-gate.mjs` is the single record of "this package lives here + * and is published elsewhere", and the cutover has to delete its entry because + * the release gate blocks every release until it does. Deleting it arms this + * pipeline. A hand-flipped flag would have nothing forcing it: the cutover + * would repoint the registries, forget the flag, and the pipeline would stay + * inert — publishing an npm package with no SQL release, docs or crate. + * + * Why not reuse the release gate: it answers a REGISTRY question about npm, and + * `release-plz.yml` publishes a CRATE on its own trigger. Keying that on the npm + * answer would also race `release.yml` on the very push that releases a version. + * This asks a question with no registry and no clock in it. */ import { appendFileSync } from 'node:fs' import process from 'node:process' import { fileURLToPath } from 'node:url' import { FROZEN_PUBLISHERS } from './release-gate.mjs' -/** The npm package whose publisher decides whether the whole line is armed. */ +/** The package whose publisher decides whether the whole line is armed. */ export const EQL_PACKAGE = '@cipherstash/eql' /** * `true` when this repository may publish the EQL release line. * - * The map is a parameter so the tests can drive both states without mutating - * the real one — and so the "armed" case is exercised now rather than first - * being exercised by the cutover. + * The map is a parameter so the tests can drive both states — the armed one + * included, rather than exercising it for the first time at the cutover. */ export function eqlPipelineArmed(frozen = FROZEN_PUBLISHERS) { return !frozen.has(EQL_PACKAGE) } -/** - * Why it is not armed, or `null` when it is. - * - * The reason is the `FROZEN_PUBLISHERS` entry itself rather than a sentence - * written here, so the job log says which repository still owns the publish and - * cannot drift from the map that decided it. - */ +/** Why it is not armed, or `null`. Taken from the map, so it cannot drift. */ export function frozenReason(frozen = FROZEN_PUBLISHERS) { return frozen.get(EQL_PACKAGE) ?? null } @@ -88,6 +55,6 @@ function main() { } } -// Importable without running, so the tests can read the two exports above -// without writing to GITHUB_OUTPUT. +// Importable without running, so the tests read the exports without writing to +// GITHUB_OUTPUT. if (process.argv[1] === fileURLToPath(import.meta.url)) main() diff --git a/scripts/lint-no-workflow-caching.mjs b/scripts/lint-no-workflow-caching.mjs index 093ea4b3b..b8d280523 100644 --- a/scripts/lint-no-workflow-caching.mjs +++ b/scripts/lint-no-workflow-caching.mjs @@ -17,19 +17,14 @@ const TARGETS = process.argv.slice(2).length // calls it too — cannot become a way to build these artifacts under // different rules. '.github/workflows/_build-ffi-artifacts.yml', - // The EQL release line. `_build-eql-sql.yml` and `_build-eql-docs.yml` - // are reached from release.yml anyway (the traversal follows a job-level - // `uses:`) and are named for the same reason `_build-ffi-artifacts.yml` - // is: so a second caller cannot become a way to build the released SQL - // and docs under different rules. + // The EQL release line. The two reusables are reached from release.yml + // anyway and named for the same reason `_build-ffi-artifacts.yml` is: a + // second caller must not become a way to build them under other rules. // - // `release-plz.yml` and `release-postgres-eql-image.yml` are NOT reached - // from anywhere — `release.yml` starts them with `gh workflow run`, which - // no traversal can follow — so naming them here is the only thing that - // brings them into scope. One publishes a crate to crates.io over OIDC; - // the other pushes a multi-arch image to GHCR. Both hold a publishing - // credential, which is the property this gate is about, and neither is - // reachable by reading `uses:` from a target. + // The other two are reached from NOWHERE — release.yml starts them with + // `gh workflow run`, which no traversal can follow — so naming them here + // is the only thing that brings them into scope. One publishes a crate + // over OIDC, the other pushes an image to GHCR. '.github/workflows/_build-eql-sql.yml', '.github/workflows/_build-eql-docs.yml', '.github/workflows/release-plz.yml', @@ -175,50 +170,37 @@ const AUDITED_ACTIONS = new Map([ // ---- The EQL release line ------------------------------------------- // - // Creates and attaches the `eql-` GitHub release from - // `_build-eql-sql.yml` and `_build-eql-docs.yml`. Read at the pinned v2: its - // inputs are release metadata and file globs, and it uploads through the - // releases API. No cache, no cache input. + // Creates and attaches the eql- GitHub release. Read at the pinned + // v2: release metadata and file globs, uploaded through the releases API. ['softprops/action-gh-release', { cacheInput: null }], - // Imports the GPG key release-plz signs its commit and tag with. Read at the - // pinned v7: key material, git config and fingerprint inputs only. + // Imports the GPG key release-plz signs with. Read at v7: key material, git + // config and fingerprint inputs only. ['crazy-max/ghaction-import-gpg', { cacheInput: null }], - // Publishes the `eql-bindings` crate. It has no cache input — AND IT IS A - // COMPOSITE, so a second sentence is owed here that the other entries do not - // need. At the pinned v0.5 it reaches three published actions of its own - // (`taiki-e/install-action`, `cargo-bins/cargo-binstall`, - // `release-plz/git-config`), and this gate CANNOT read them: it opens local - // `./` composites and nothing else. So this entry is an audit of the version - // pinned above, not a standing property of the action — read the composite - // again when the pin moves. It was read at 2eb1d8bc: the three nested actions - // download binaries (GitHub releases and crates.io) and none of them touches - // the GitHub Actions cache. + // Publishes the eql-bindings crate. No cache input — AND IT IS A COMPOSITE + // reaching three published actions of its own (taiki-e/install-action, + // cargo-bins/cargo-binstall, release-plz/git-config) that this gate cannot + // read; it opens local `./` composites only. So this is an audit of the + // pinned version, not a standing property: re-read it when the pin moves. + // Read at 2eb1d8bc — all three download binaries, none touches the cache. ['release-plz/action', { cacheInput: null }], // ---- Docker, for the postgres-eql image ------------------------------ // - // The plan for these three said their caching was "registry-backed rather - // than GitHub-Actions-cache", so the allowlist's rule would not transfer. - // Reading the pinned action manifests, that is true of exactly ONE of them. + // The plan called all three "registry-backed rather than + // GitHub-Actions-cache". Reading the pinned manifests, that is true of one. // - // `cache-image` DEFAULTS TO TRUE and stores the binfmt image in the GitHub - // Actions cache — the same shape as `jdx/mise-action` above, and an omitted - // key is a restore. + // `cache-image` DEFAULTS TO TRUE — binfmt image in the Actions cache. ['docker/setup-qemu-action', { cacheInput: 'cache-image' }], - // `cache-binary` DEFAULTS TO TRUE and caches the buildx binary the same way. + // `cache-binary` DEFAULTS TO TRUE — the buildx binary, same way. ['docker/setup-buildx-action', { cacheInput: 'cache-binary' }], - // Authenticates to GHCR. No cache, no cache input. + // GHCR auth. No cache, no cache input. ['docker/login-action', { cacheInput: null }], - // THE ONE THE PLAN DESCRIBED, and the only entry that needs - // `forbiddenInputs`. It has no boolean that turns caching off, because it has - // no caching of its own to turn off: buildx caches only what `cache-from` / - // `cache-to` ask it to. Those are usually registry-backed - // (`type=registry,ref=…`), which is outside this gate's concern — but - // `type=gha` names the GitHub Actions cache directly, and would be a restore - // AND a write inside the job that pushes the published image. `cacheInput` - // cannot express that: there is no input to set to `false`. So the two inputs - // are forbidden outright. Adding one back means deciding here, in this file, - // which backend it names. + // The one the plan described, and the only `forbiddenInputs` entry. It has no + // boolean to turn caching off because it has none of its own: buildx caches + // what `cache-from` / `cache-to` ask for. Usually registry-backed, which is + // outside this gate — but `type=gha` is the Actions cache, restored AND + // written inside the job that pushes the image. Forbidden outright; adding + // one back means deciding here which backend it names. [ 'docker/build-push-action', { cacheInput: null, forbiddenInputs: ['cache-from', 'cache-to'] }, @@ -382,14 +364,11 @@ function checkStep(step, at, bodyAudited = false) { if (reason) offenders.push(`${at}: ${path} ${reason}`) } - // The other half of the same audit, for an action whose caching is opted - // INTO rather than defaulted on. `cacheInput` asks for an explicit `false`; - // there is nothing to set to `false` here, so the requirement is that the - // input is absent. Presence is the finding, whatever its value — the value is - // a backend name (`type=gha` restores and writes the GitHub Actions cache, - // `type=registry` does not), and deciding which is which by string-matching a - // buildx cache spec would be a parser this file does not want to own. Naming - // one at all is the moment to record the decision on the allowlist entry. + // The same audit for an action whose caching is opted INTO rather than + // defaulted on: nothing to set to `false`, so the input must be absent. + // Presence is the finding whatever the value — the value is a backend name, + // and telling `type=gha` from `type=registry` would mean owning a parser for + // buildx cache specs. Naming one is the moment to record the decision above. for (const forbidden of audited?.forbiddenInputs ?? []) { if (step?.with && Object.hasOwn(step.with, forbidden)) { offenders.push( From 77aefd3a3b14e5d20d63b409c9460403d2de9a39 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 25 Aug 2026 12:30:27 +1000 Subject: [PATCH 3/5] fix(eql): act on the differential audit against encrypt-query-language --- .github/workflows/_build-eql-docs.yml | 6 + .github/workflows/_build-eql-sql.yml | 6 + .github/workflows/rebuild-docs.yml | 33 --- .github/workflows/release-plz.yml | 55 ++++- .github/workflows/release.yml | 64 +++--- AGENTS.md | 193 +++++++++--------- SECURITY.md | 38 ++-- .../2026-08-13-eql-monorepo-absorption.md | 13 +- package.json | 2 +- packages/eql/DEVELOPMENT.md | 25 +-- packages/eql/docker/Dockerfile | 2 +- .../eql/scripts/verify-release-assets.mjs | 26 ++- packages/eql/release-plz.toml | 11 +- .../__tests__/cargo-lock-freshness.test.mjs | 1 + .../__tests__/cargo-publish-opt-out.test.mjs | 86 +++++--- scripts/__tests__/eql-pipeline-armed.test.mjs | 1 - scripts/__tests__/eql-suite-ci.test.mjs | 21 +- scripts/__tests__/lib/package-readmes.mjs | 43 ++++ .../no-removed-drizzle-surface.test.mjs | 13 +- .../no-removed-eql-version-flag.test.mjs | 13 +- .../workflow-paths-filter-parity.test.mjs | 14 ++ scripts/lint-no-eql-registry-pins.mjs | 10 + 22 files changed, 425 insertions(+), 251 deletions(-) delete mode 100644 .github/workflows/rebuild-docs.yml create mode 100644 scripts/__tests__/lib/package-readmes.mjs diff --git a/.github/workflows/_build-eql-docs.yml b/.github/workflows/_build-eql-docs.yml index bd12aa93f..63e6fb4ca 100644 --- a/.github/workflows/_build-eql-docs.yml +++ b/.github/workflows/_build-eql-docs.yml @@ -78,6 +78,12 @@ jobs: path: | packages/eql/release/eql-docs-*.zip packages/eql/release/eql-docs-*.tar.gz + # `warn` is the default, and a warning here is indistinguishable from + # a successful upload on the run page. These paths are the one thing + # the subtree import could plausibly get wrong — they are workspace- + # root relative and so are NOT covered by `defaults.run.working- + # directory` — and a wrong one publishes a release with no assets. + if-no-files-found: error - name: Publish documentation to release if: ${{ inputs.tag != '' }} diff --git a/.github/workflows/_build-eql-sql.yml b/.github/workflows/_build-eql-sql.yml index 99d5f72f6..d426bf9b8 100644 --- a/.github/workflows/_build-eql-sql.yml +++ b/.github/workflows/_build-eql-sql.yml @@ -92,6 +92,12 @@ jobs: path: | packages/eql/release/cipherstash-encrypt.sql packages/eql/release/cipherstash-encrypt-uninstall.sql + # `warn` is the default, and a warning here is indistinguishable from + # a successful upload on the run page. These paths are the one thing + # the subtree import could plausibly get wrong — they are workspace- + # root relative and so are NOT covered by `defaults.run.working- + # directory` — and a wrong one publishes a release with no assets. + if-no-files-found: error - name: Attach artefacts to existing release if: ${{ inputs.attach && inputs.target_commitish == '' }} diff --git a/.github/workflows/rebuild-docs.yml b/.github/workflows/rebuild-docs.yml deleted file mode 100644 index 888cc3f29..000000000 --- a/.github/workflows/rebuild-docs.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Rebuild Docs - -# ⚠️ THIS WORKFLOW HAS NEVER RUN. `gh run list --workflow rebuild-docs.yml` is -# empty, across @cipherstash/stack@1.0.0, 1.1.0 and 1.1.1. A ref created with -# the repository's GITHUB_TOKEN does not start a workflow run, and -# changesets/action creates these tags with that token — so the `push` event -# below is never delivered. Valid file, configured secret, existing tags, and -# the docs site rebuilt by nothing. -# -# Left as-is because fixing it means changing the JS release path, which the -# pull request that discovered it was not doing. DO NOT ADD A TAG PATTERN HERE -# expecting it to fire. The EQL half was routed around it: it is now the -# `eql-docs-rebuild` job in release.yml, firing the webhook from inside the run -# that published. The same fix applies here. -on: - push: - tags: - - '@cipherstash/stack@*' - -permissions: - contents: read - -jobs: - trigger-docs-rebuild: - name: Trigger Docs Rebuild - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Send webhook - env: - WEBHOOK_URL: ${{ secrets.DOCS_WEBHOOK_URL }} - run: | - curl -X POST "$WEBHOOK_URL" diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index 511126af6..f9df54819 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -12,9 +12,11 @@ name: "Release eql-bindings (crates.io)" # crate release does not fire the `eql-*` SQL/docs/image automation. # # INERT until the Phase-5 cutover: the `release` job is gated on -# scripts/eql-pipeline-armed.mjs, which reads FROZEN_PUBLISHERS. Before arming -# it, this repository also needs a GPG_PRIVATE_KEY secret (it has none) and a -# crates.io Trusted Publishing entry naming cipherstash/stack + release-plz.yml. +# scripts/eql-pipeline-armed.mjs, which reads FROZEN_PUBLISHERS. Before the +# first crate release this repository also needs a GPG_PRIVATE_KEY secret (it +# has none) and a crates.io Trusted Publishing entry naming cipherstash/stack + +# release-plz.yml. Neither is needed for a run that publishes nothing: the +# preflight step decides that before the key is imported. # Read-only by default; the publishing job escalates for itself. Enforced by # scripts/__tests__/workflow-publish-permissions.test.mjs. @@ -25,6 +27,20 @@ on: push: branches: - main + # Scoped to the subtree. Upstream had no filter and did not need one: it was + # an EQL-ONLY repository, where "every push to main" and "every EQL change" + # were the same set. Here they are not, and this job holds crates.io OIDC, + # `contents: write` and the signing key — booting it on every merge to a + # monorepo is a standing credential exposure for no release. + # + # A release that moves the crate always matches: `changeset version` runs + # scripts/sync-lockstep-versions.mjs, which rewrites + # packages/eql/crates/eql-bindings/Cargo.toml and the SQL assets beside it. + # Kept in step with the other EQL filters by + # scripts/__tests__/eql-workflow-filters.test.mjs. + paths: + - "packages/eql/**" + - ".github/workflows/release-plz.yml" workflow_dispatch: {} # Never cancel — a cancelled release can leave a half-published state. @@ -72,14 +88,6 @@ jobs: with: fetch-depth: 0 - - name: Import GPG key - uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 - with: - gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - git_user_signingkey: true - git_commit_gpgsign: true - git_tag_gpgsign: true - # No mise step (upstream had one): nothing here runs a mise task, and # release-plz needs only cargo and git, which the runner image ships. @@ -88,6 +96,15 @@ jobs: # committed tree verbatim, so a hand-pinned Cargo.toml would ship the # DEV placeholder as its "exact SQL". Only enforced when this run would # actually publish; a crates.io API failure fails towards enforcing. + # + # ALSO decides whether the GPG import below runs. This step used to sit + # BELOW the key import, which meant a repository with no GPG_PRIVATE_KEY + # failed here on every push to main the moment the pipeline was armed — + # including pushes that release nothing. Nothing published (the import + # fails closed), but main went permanently red with an error that reads + # like a broken release rather than a missing secret. Ordering the + # registry question first is what makes a no-op run a no-op. + id: preflight working-directory: packages/eql run: | set -euo pipefail @@ -97,8 +114,12 @@ jobs: published="$(curl -fsSL --retry 3 -H "User-Agent: cipherstash-eql-release (https://github.com/cipherstash/stack)" "https://crates.io/api/v1/crates/eql-bindings/versions" | jq -r '.versions[].num' | grep -Fx "$cargo_version" || true)" if [[ -n "$published" ]]; then echo "eql-bindings@${cargo_version} is already on crates.io; publish no-op — skipping the bundled-SQL guard" + echo "publish_needed=false" >> "$GITHUB_OUTPUT" exit 0 fi + # Empty `published` also covers a crates.io API failure, so this is + # the fail-towards-enforcing branch: guard the SQL, import the key. + echo "publish_needed=true" >> "$GITHUB_OUTPUT" manifest_version="$(jq -r '.eqlVersion' crates/eql-bindings/sql/release-manifest.json)" if [[ "$manifest_version" != "$cargo_version" ]]; then echo "::error::release-manifest.json eqlVersion ('$manifest_version') does not match Cargo.toml ('$cargo_version'). Run 'pnpm run version' from the repository root." >&2 @@ -115,6 +136,18 @@ jobs: exit 1 fi + - name: Import GPG key + # release-plz signs the commit and tag it creates. When the preflight + # says the version is already published it creates neither, so there is + # nothing to sign and no reason to require the secret. + if: steps.preflight.outputs.publish_needed == 'true' + uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + git_user_signingkey: true + git_commit_gpgsign: true + git_tag_gpgsign: true + - name: Run release-plz release uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8da0530d7..6b20d7969 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,10 +95,14 @@ jobs: # Two levels down: the subtree root carries no package.json. # Upstream this was `./packages/eql/package.json`. version="$(node -p "require('./packages/eql/packages/eql/package.json').version")" - # The same regex prepare-bindings-assets.sh enforces, because this - # job runs first and that script runs last. Upstream checked only - # for a hyphen, so `3.0.6-beta` classified as a prerelease and got - # a public tag and GitHub release before the npm job died on it. + # prepare-bindings-assets.sh enforces the same shape with the + # suffix OPTIONAL; here it is required, because a version reaching + # this branch has already been classified a prerelease and a bare + # `3.0.6` would be one wearing the wrong identity. Checked here + # because this job runs first and that script runs last. Upstream + # checked only for a hyphen, so `3.0.6-beta` classified as a + # prerelease and got a public tag and GitHub release before the npm + # job died on it. if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then echo "::error::a prerelease release commit must pin an exact prerelease identity (X.Y.Z-(alpha|beta|rc).N) in packages/eql/packages/eql/package.json; found '${version}'" >&2 exit 1 @@ -565,35 +569,6 @@ jobs: ref: ${{ github.sha }} tag: eql-${{ needs.release.outputs.eql_version }} - eql-docs-rebuild: - name: Rebuild the docs site - needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] - # Not a tag trigger, which is why this job exists. Upstream used - # `on: push: tags: ['eql-*']`, but the eql- tag is created by - # softprops/action-gh-release with GITHUB_TOKEN, and a ref created with that - # token starts no workflow run — the same rule `eql-image` works around, and - # the reason rebuild-docs.yml has never run. Firing from inside the - # publishing run removes the trigger from the picture. - if: >- - needs.classify.outputs.mode == 'production' && - needs.eql-armed.outputs.armed == 'true' && - needs.release.outputs.eql_published == 'true' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Send webhook - env: - WEBHOOK_URL: ${{ secrets.DOCS_WEBHOOK_URL }} - VERSION: ${{ needs.release.outputs.eql_version }} - run: | - set -euo pipefail - if [ -z "$WEBHOOK_URL" ]; then - echo "::error::DOCS_WEBHOOK_URL is not configured; the docs site will not rebuild for eql-${VERSION}" >&2 - exit 1 - fi - curl -X POST --fail-with-body "$WEBHOOK_URL" - echo "docs rebuild requested for eql-${VERSION}" - eql-image: name: Dispatch the Postgres + EQL image build needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] @@ -818,9 +793,17 @@ jobs: needs: - classify - eql-armed + # `gate` and `release` are not EQL jobs, and they are the two that most + # often decide an EQL run does nothing. `gate` exits non-zero for a frozen + # publisher — which is the NORMAL inert state — and skips `release`, whose + # `eql_published` output then gates the production EQL chain. Without them + # here every job below reads `skipped` and the table cannot separate + # "correctly inert" from "the gate refused this release" from "changesets + # failed", which is the distinction this job exists to draw. + - gate + - release - eql-sql - eql-docs - - eql-docs-rebuild - eql-image - prerelease-eql-sql - prerelease-eql-docs @@ -835,9 +818,10 @@ jobs: MODE: ${{ needs.classify.outputs.mode }} VERSION: ${{ needs.classify.outputs.version }} ARMED: ${{ needs.eql-armed.outputs.armed }} + GATE: ${{ needs.gate.result }} + RELEASE: ${{ needs.release.result }} EQL_SQL: ${{ needs.eql-sql.result }} EQL_DOCS: ${{ needs.eql-docs.result }} - EQL_DOCS_REBUILD: ${{ needs.eql-docs-rebuild.result }} EQL_IMAGE: ${{ needs.eql-image.result }} PRE_SQL: ${{ needs.prerelease-eql-sql.result }} PRE_DOCS: ${{ needs.prerelease-eql-docs.result }} @@ -854,12 +838,20 @@ jobs: echo "" echo "| job | result |" echo "| --- | --- |" + echo "| gate | ${GATE} |" + echo "| release (changesets) | ${RELEASE} |" echo "| eql-sql | ${EQL_SQL} |" echo "| eql-docs | ${EQL_DOCS} |" - echo "| eql-docs-rebuild | ${EQL_DOCS_REBUILD} |" echo "| eql-image | ${EQL_IMAGE} |" echo "| prerelease-eql-sql | ${PRE_SQL} |" echo "| prerelease-eql-docs | ${PRE_DOCS} |" echo "| prerelease-eql-npm | ${PRE_NPM} |" echo "| prerelease-eql-crate | ${PRE_CRATE} |" + if [[ "${ARMED}" != "true" ]]; then + echo "" + echo "The EQL pipeline is INERT: \`@cipherstash/eql\` is a frozen publisher in \`scripts/release-gate.mjs\`, so every job above is expected to be \`skipped\`. Deleting that entry is what arms it — see the Phase 5 cutover in \`docs/plans/2026-08-13-eql-monorepo-absorption.md\`." + elif [[ "${GATE}" != "success" ]]; then + echo "" + echo "The pipeline is armed but \`gate\` did not succeed, so nothing downstream of it ran. Run \`node scripts/release-gate.mjs\` for what it is refusing." + fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index dc85d0a67..fd1555778 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -319,24 +319,48 @@ monorepo, which is where the silent failures are. | `_build-eql-sql.yml`, `_build-eql-docs.yml` | `workflow_call` reusables, called from both EQL paths so there is one build path per artefact | | `release-plz.yml` | the crate. **Its filename cannot change** — crates.io Trusted Publishing binds to it | | `release-postgres-eql-image.yml` | the GHCR image, dispatched by `release.yml` on production finals | - | `rebuild-docs.yml`, `lint-release.yml` | merged into the root files of the same name | - - Every one sits behind `scripts/eql-pipeline-armed.mjs`, which reads - `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`. **Deleting the - `@cipherstash/eql` entry at the Phase-5 cutover is what arms the pipeline** — - there is no second flag, because a forgotten one fails silently in the - direction that publishes an npm package with no SQL release, docs or crate. - Run `node scripts/release-gate.mjs` for what the gate is doing right now - rather than inferring it from here. - - **The cutover needs three more things, none of which a workflow can assert + | `lint-release.yml` | merged into the root file of the same name | + | ~~`rebuild-docs.yml`~~ | **not ported.** It targeted the retired docs site through the deprecated `DOCS_WEBHOOK_URL`; versioned docs artifacts are still built by `_build-eql-docs.yml` | + + **Inertness is a derived switch, not a flag somebody flips.** The one piece + of state is `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs` — the existing + map recording "this package lives here but is published elsewhere" — and + every job that publishes an EQL artefact is gated on its answer. **Deleting + the `@cipherstash/eql` entry at the Phase-5 cutover is what arms the + pipeline**, and there is no second thing to remember, because the cutover has + to delete it anyway: the release gate blocks every release until it does. A + separate flag would have failed silently in the worst direction — an npm + package published with no SQL release, no docs and no crate. + + **Two readers, one map, and they are not the same code path.** Say which you + mean: + + - `scripts/eql-pipeline-armed.mjs` imports the map and answers "may this + repository publish EQL at all?". Ten jobs across `release.yml`, + `release-plz.yml` and `release-postgres-eql-image.yml` carry + `if: needs.eql-armed.outputs.armed == 'true'`. That covers the SQL bundle, + the docs, the crate, the image and the *prerelease* npm publish. + - The **production npm publish does not.** `release.yml`'s `release` job runs + `changeset publish` gated only on `needs.gate.result == 'success'`, and + `.changeset/config.json` has `"ignore": []`, so `@cipherstash/eql` is + publishable like any other workspace member. What holds it is + `release-gate.mjs` exiting non-zero for a frozen publisher, which fails the + `gate` job and skips `release` entirely. + + Both keyed on the same entry, so one deletion still arms all five. But + nothing asserts the two agree: relax the gate and the npm half opens while + `eql-armed` still reports `false` and the other four jobs skip correctly — + which is exactly the outcome the derived switch exists to prevent. This + paragraph said "every one sits behind `eql-pipeline-armed.mjs`" until an + audit read the `needs:` list. Run `node scripts/release-gate.mjs` for what + the gate is doing right now rather than inferring it from here. + + **The cutover needs two more things, neither of which a workflow can assert ahead of time:** a `GPG_PRIVATE_KEY` secret (release-plz signs its commit and tag, and this repository has none); write access from here to the `ghcr.io/cipherstash/postgres-eql` package, still linked to - `cipherstash/encrypt-query-language`; and a docs-site rebuild that actually - fires — `rebuild-docs.yml` has never run in its life, because a ref created - with `GITHUB_TOKEN` starts no workflow. The EQL half is routed around it as a - job in `release.yml`; the `@cipherstash/stack@*` half is still broken. + `cipherstash/encrypt-query-language`. The old docs-site webhook is not a + cutover dependency: that site and `DOCS_WEBHOOK_URL` are retired. - **A release workflow may not restore a cache, including a Rust one.** `lint-no-workflow-caching.mjs` forbids a cache restore anywhere an artefact is published; `eql-suite-ci.test.mjs` wants `Swatinem/rust-cache` on every job @@ -468,86 +492,73 @@ monorepo, which is where the silent failures are. `CS_WORKSPACE_CRN` re-pins both. `tests/sqlx/src/selectors.rs` holds five more workspace-keyed constants with no consumers and no guard — delete or guard them before using any of them. -- **Publishing has not moved yet.** npm trusted publishing for `@cipherstash/eql` - still names `cipherstash/encrypt-query-language` — the SLSA provenance on - `@cipherstash/eql@3.0.5` records that repository and - `.github/workflows/release.yml` — and the package's own `repository` / `bugs` - fields still point there, as do the `eql-bindings` crate's. Repointing all of - it is the Phase-5 cutover. **The pipeline it repoints at already exists** — - see "The EQL release pipeline is built and INERT" above; what is left is the - registry-side move plus deleting the map entry that keeps it inert. - This was protect-ffi's situation until its own cutover, with one difference: - the guard is not a changeset lint but `scripts/release-gate.mjs`, and it is - the stronger of the two. Its `FROZEN_PUBLISHERS` map lists every package that lives here but is - published from another repository, and the gate **exits non-zero** — failing - the `gate` job, which skips `release` entirely — if such a package's committed - version is missing from npm, if any published package carries a runtime - `workspace:` range that only that package could satisfy, or — the third - check, added after a review caught it by hand — if a frozen package's - in-tree artefact is **not the bytes published under the version the tree - claims**. Any one stops the Version Packages PR and the publish alike. A - changeset-side guard sees none of them — which is why the hand-applied 3.0.5 - bump needed this one. - - **That third check is the one worth understanding before you touch - `packages/eql`.** For a package this repo publishes, in-tree bytes differing - from npm is an unreleased change — every pull request. For a frozen one it is - a contradiction: the version cannot be released from here, so the tree is not - proposing those bytes, it is *asserting they are already on npm under that - number*. Nothing local can notice when that stops being true, because - `sql/release-manifest.json` is regenerated with the SQL and goes on agreeing - with it; only the registry disagrees. This branch shipped exactly that — a - `3.0.5` subtree whose install bundle hashed `7ad9c9f8…` against npm's - `accde0030…`, because upstream restored the deprecated `ste_vec_contains` - aliases in the real release — and `stash eql install` would have executed that - SQL against a customer database, which would then have carried functions the - version it reports does not define. (The CLI now refuses a bundle whose bytes - do not hash to its own release manifest — see - `packages/cli/src/installer/bundle-digest.ts`. That catches a corrupt or - tampered `node_modules`, **not** this: a frozen-package skew regenerates the - manifest alongside the SQL, so the two agree locally and only the registry - disagrees. The release gate is still the only thing that notices.) The check - `npm pack`s the frozen package - and compares the two release manifests; `FROZEN_ARTEFACT_DIGESTS` says which - artefact, keyed identically to `FROZEN_PUBLISHERS` and deleted with it at the - cutover. - **That `npm pack` + `tar` extraction was itself executed by no test until the - #915 follow-up** — every `frozenBytesSkew` unit test injected both digests, - and the end-to-end process test shimmed `npm` with a script answering `npm - view` only, so `npm pack` got a synthetic `E404` and the comparison was - skipped on every run. Driving it for real found two defects in it: `--silent` - suppressed the very error text the function classified on, and npm answers a - missing *version* of an existing package with `ETARGET`, not `E404` — so - `@cipherstash/eql@`, the exact case the function documents, became an - uncaught throw with an empty reason, raised while building the blocker array, - which meant `reportBlockers` never printed the actionable `frozen-publisher` - remedy it had already computed. +- **`FROZEN_PUBLISHERS` is what stops this repo publishing EQL, and it is a + mechanism, not a status.** `scripts/release-gate.mjs` carries a map of every + package that lives here but is published from another repository. For each, + the gate **exits non-zero** — failing the `gate` job, which skips `release` + entirely — on any of three conditions: + + 1. the package's committed version is missing from npm; + 2. any published package carries a runtime `workspace:` range that only that + package could satisfy; + 3. the frozen package's in-tree artefact is **not the bytes published under + the version the tree claims**. + + Any one stops the Version Packages PR and the publish alike. A changeset-side + guard sees none of them. + **Whether the gate is blocking anything right now is a question for the registry, not for this file: run `node scripts/release-gate.mjs` and read what it says.** `tests.yml` runs the same script at PR time so the answer arrives a - merge earlier. `@cipherstash/eql` is the map's ONLY entry: the seven - protect-ffi packages were listed there too, and the cutover that repointed - their publisher at this repo did not take them out — which left the gate - armed against the first release that cutover had just enabled. Delete the - `@cipherstash/eql` entry in the Phase-5 cutover — + merge earlier. Which registry publishes what, and whether trusted publishing + has been repointed, is likewise configuration — check the registry, do not + read it here. This bullet used to narrate that state and was wrong twice. + + **Delete the `@cipherstash/eql` entry in the Phase-5 cutover.** `scripts/__tests__/frozen-publisher-docs.test.mjs` fails until this paragraph - goes with it, and `release-gate.test.mjs` now asserts the map carries no FFI - name, so that particular mistake cannot be made twice. - **`SECURITY.md`'s "Note on publishing" is the third document that guard - holds**, added after it was found still naming the seven - `@cipherstash/protect-ffi*` packages as published from `protectjs-ffi` — a - sentence the FFI cutover made false and nothing checked, in the one file that - tells a reporter which pipeline built the artefact they are reporting on. The - guard now fails on a doc that freezes a package the map does not, so the same - drift cannot outlive the next cutover either. - Note too that 3.0.5 did *not* come from `changeset version` — - eleven unrelated changesets were pending, so the bump was entered by hand in - `packages/eql/packages/eql/CHANGELOG.md` and the parked - `rename-ste-vec-contains.md.deferred` deleted with it, precisely so the - cutover cannot apply the same bump twice. Read that CHANGELOG entry before - assuming a version's provenance. -- **Changesets for `@cipherstash/eql` go in the repo-root `.changeset/`.** - `packages/eql/.changeset/` is an empty leftover of the subtree. + goes with it, and `SECURITY.md`'s "Note on publishing" is the third document + it holds — the one file that tells a reporter which pipeline built the + artefact they are reporting on. `release-gate.test.mjs` also asserts the map + carries no FFI name: the seven protect-ffi packages were left in it after + their own cutover, which armed the gate against the first release that + cutover had just enabled. + + **Check 3 is the one worth understanding before you touch `packages/eql`.** + For a package this repo publishes, in-tree bytes differing from npm is an + unreleased change — every pull request. For a frozen one it is a + contradiction: the version cannot be released from here, so the tree is not + proposing those bytes, it is *asserting they are already on npm under that + number*. Nothing local can notice when that stops being true, because + `sql/release-manifest.json` is regenerated with the SQL and goes on agreeing + with it; only the registry disagrees. That has happened, and the SQL would + have reached a customer database through `stash eql install`, carrying + functions the version it reports does not define. + + The CLI now refuses a bundle whose bytes do not hash to its own release + manifest (`packages/cli/src/installer/bundle-digest.ts`), but that catches a + corrupt or tampered `node_modules`, **not** this — a frozen-package skew + regenerates the manifest alongside the SQL, so the two agree locally. The + release gate is still the only thing that notices. It `npm pack`s the frozen + package and compares the two release manifests; `FROZEN_ARTEFACT_DIGESTS` + says which artefact, keyed identically to `FROZEN_PUBLISHERS` and deleted + with it. + + That `npm pack` path was itself executed by **no test** until the #915 + follow-up: every unit test injected both digests and the process test shimmed + `npm` to answer `npm view` only. Driving it for real found two defects — + `--silent` suppressed the error text the function classified on, and npm + answers a missing *version* of an existing package with `ETARGET`, not + `E404`, so the exact case the function documents became an uncaught throw + that swallowed the actionable remedy. Assume a guard is untested until you + have watched it fail. + + Note also that `@cipherstash/eql@3.0.5` did not come from `changeset + version`: eleven unrelated changesets were pending, so the bump was entered + by hand in `packages/eql/packages/eql/CHANGELOG.md`. Read that CHANGELOG + entry before assuming a version's provenance. +- **Changesets for `@cipherstash/eql` go in the repo-root `.changeset/`.** The + subtree's own `.changeset/` was deleted with the import; there is no second + one to put them in by mistake. ## Agent Skills — these ship to customers diff --git a/SECURITY.md b/SECURITY.md index 6896ad615..eab646082 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,24 +27,28 @@ lockstep with `@cipherstash/eql`. It is in scope for security reports on the same terms as the npm packages above. > **Note on publishing.** `@cipherstash/eql` and the `eql-bindings` crate are -> developed here but are still *published* from -> `cipherstash/encrypt-query-language` — the npm package's `repository` / -> `bugs` fields and the crate's `repository` / `homepage` all still name it, as -> does npm trusted publishing, and repointing every one of them is Phase 5 of -> `docs/plans/2026-08-13-eql-monorepo-absorption.md`. Everything else in the -> table above — including all seven `@cipherstash/protect-ffi*` packages, whose -> own cutover has completed — is published from this repository by -> `.github/workflows/release.yml`. Source, issues, and security reports for all -> of them belong here regardless. +> developed here but are *published* from `cipherstash/encrypt-query-language` +> until the Phase 5 cutover in +> `docs/plans/2026-08-13-eql-monorepo-absorption.md` completes. Everything else +> in the table above, including all seven `@cipherstash/protect-ffi*` packages, +> is published from this repository by `.github/workflows/release.yml`. +> **Source, issues, and security reports for all of them belong here +> regardless** — that part does not depend on which pipeline built the +> artefact. > -> The provenance attestation on a release names the repository that built it, -> and is the only thing that settles the paragraph above: -> `curl -s https://registry.npmjs.org/-/npm/v1/attestations/@cipherstash%2fprotect-ffi@0.32.0` -> returns `cipherstash/stack`; the same call against `@cipherstash%2feql@3.0.5` -> returns `cipherstash/encrypt-query-language`. Check there before repeating -> either claim: this note went on naming `cipherstash/protectjs-ffi` as the -> protect-ffi publisher through `0.32.0` — the release that proved the cutover -> and disproved the sentence. +> **Do not trust this note for which repository published a given release.** +> Registry configuration changes without touching this file, and this note has +> been wrong before: it named `cipherstash/protectjs-ffi` as the protect-ffi +> publisher through `0.32.0`, the release that disproved the sentence. The +> provenance attestation is the authority, and it is per-release: +> +> ``` +> curl -s https://registry.npmjs.org/-/npm/v1/attestations/@cipherstash%2f@ +> ``` +> +> Read the `workflow.repository` and `workflow.path` it returns. That names the +> repository and workflow that actually built the artefact you are reporting +> on. > > `scripts/__tests__/frozen-publisher-docs.test.mjs` now holds this paragraph to > `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`. It fails if the note names diff --git a/docs/plans/2026-08-13-eql-monorepo-absorption.md b/docs/plans/2026-08-13-eql-monorepo-absorption.md index efa8d9124..cffeaf553 100644 --- a/docs/plans/2026-08-13-eql-monorepo-absorption.md +++ b/docs/plans/2026-08-13-eql-monorepo-absorption.md @@ -209,7 +209,7 @@ EQL ships everything at a single version V, computed by Changesets from `package | `@cipherstash/eql` (npm) | Changesets, `release.yml` | npm OIDC, bound to `release.yml` | | `eql-bindings` (crates.io) | release-plz, `release-plz.yml` | crates.io OIDC, bound to `release-plz.yml` | | SQL install/uninstall bundle | `_build-sql.yml`, attached to the `eql-` GitHub release | — | -| API docs (HTML/XML/Markdown) | `_build-docs.yml` + `rebuild-docs.yml` webhook to Vercel | `DOCS_WEBHOOK_URL` secret | +| API docs (HTML/XML/Markdown) | `_build-docs.yml`, attached to the `eql-` GitHub release | — | | `ghcr.io/cipherstash/postgres-eql` | `release-postgres-eql-image.yml` | `GITHUB_TOKEN` | plus **dbdev**, which is documented as manual and lagging. @@ -320,13 +320,16 @@ Five surfaces, each flipped independently, each inert until its own cutover. - [x] **Carried over from Phase 4, deliberately.** Port `lint-release.yml` and the two reusable workflows (`_build-sql.yml`, `_build-docs.yml`) alongside the release workflows they gate and are called from — a lint aimed at four absent files is worse than no lint. Add `cache: false` to the three `jdx/mise-action` steps that omit it (`_build-sql.yml`, `_build-docs.yml`, `release-postgres-eql-image.yml`), and add those two plus `release-plz.yml` to `scripts/lint-no-workflow-caching.mjs`'s `TARGETS`, with their remote actions in `AUDITED_ACTIONS` — or the gate fails on the addition itself. **Two corrections from doing it.** The three files did not omit `cache:`; they set it to `true`, one hit each, so the fix was to flip the value rather than add a key (adding one produces a duplicate YAML key). And the reusables were renamed on the way in — `_build-eql-sql.yml`, `_build-eql-docs.yml` — because "SQL" and "docs" name nothing on their own in a monorepo and nothing binds to either filename; `release-plz.yml` is the one that could not be renamed. `release-plz.yml` also lost its mise step entirely: no step in that job runs a mise task, and the publishing-job rule (`add_shims_to_path: false`) would have left mise's toolchain off PATH anyway. - [x] **Then delete `packages/eql/.github/` entirely**, and convert the shrinking allowlist in `scripts/__tests__/eql-suite-ci.test.mjs` to a straight `existsSync(...) === false`, alongside the existing protect-ffi assertion in `lintWiring.test.ts`. The allowlist is an equality, so it will already be failing by the time the last file goes — that is what makes this step forced rather than remembered. - [x] ~~Add `scripts/lint-no-eql-changeset.mjs` in the shape of the FFI guard~~ — **superseded, and do not restore the parked changeset.** The guard landed instead as `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs`, which is strictly stronger: it exits non-zero on any frozen package whose committed version is missing from npm *and* on any runtime `workspace:` range only that package could satisfy, a pair of conditions no changeset-side guard can see — which is why the hand-applied 3.0.5 bump needed this guard rather than that one. (Run `node scripts/release-gate.mjs` for what it blocks at any given moment; that answer comes from the registry and is not recorded here.) It runs at PR time via `tests.yml` and at release time via `release.yml`. `.changeset/rename-ste-vec-contains.md.deferred` was deleted deliberately in `9b1c44d9` — the 3.0.5 bump was entered by hand in the CHANGELOG, so re-parking that `major` changeset would make the cutover bump the package a second time (3.0.5 → 4.0.0) for a rename already released. **Phase-5 action: delete the `@cipherstash/eql` entry from `FROZEN_PUBLISHERS`, nothing more.** -- [ ] Repoint all EQL manifests at `cipherstash/stack`: `packages/eql/packages/eql/package.json` — `repository.url`, `repository.directory` → `packages/eql/packages/eql`, **and `bugs.url`** (line 13, still `encrypt-query-language/issues`, which would send users to an archived repo) — plus `crates/eql-bindings/Cargo.toml` (`repository`, `homepage`) and `release-plz.toml`'s prose. npm and crates.io both reject a publish whose repository field does not match; `bugs.url` is not enforced, which is why it is the one that survives a cutover unnoticed. +- [x] Repoint all EQL manifests at `cipherstash/stack` (`e52d3317`): `packages/eql/packages/eql/package.json` — `repository.url`, `repository.directory` → `packages/eql/packages/eql`, **and `bugs.url`** (line 13, still `encrypt-query-language/issues`, which would send users to an archived repo) — plus `crates/eql-bindings/Cargo.toml` (`repository`, `homepage`) and `release-plz.toml`'s prose. npm and crates.io both reject a publish whose repository field does not match; `bugs.url` is not enforced, which is why it is the one that survives a cutover unnoticed. - [x] Port `release.yml`'s EQL half into this repo's `release.yml` — the existing gate-then-publish structure already models "one version authority plus N idempotent publishers", which is the shape it was derived from. Preserve the `classify` prerelease path and the `eql-typescript-v` idempotency short-circuit. **What it cost, all of it found by this repo's guards rather than by review.** `workflow_dispatch` on `release.yml` (the prerelease path is dispatched) means every job in the file is reachable by hand, so `classify` gates the FFI and changesets jobs too. Three guards had to grow: `workflow-publish-permissions.test.mjs` needed `REPO_WRITE_JOBS` beside `OIDC_JOBS`, because six new jobs need a writable scope and must NOT be able to publish; `workflow-dispatch-job-conditions.test.mjs` needed a reasoned skip list, because the production and prerelease halves are mutually exclusive by construction and no single synthetic context can run both; and `eql-suite-ci.test.mjs`'s rust-cache requirement needed a publish-path exemption derived from the caching linter's own targets. Two real defects came out of the same pass: the prerelease job needed `Install node-gyp` before `pnpm install` (node-pty, which upstream did not have) and `pnpm exec turbo run build` rather than `pnpm --filter … build` (`^build` dependencies, which upstream did not have either). - [x] Port `release-plz.yml` unchanged in filename — crates.io trusted publishing binds to it. It also needed `manifest_path: packages/eql/Cargo.toml` and `config: packages/eql/release-plz.toml`, which upstream did not: a composite action's steps do not inherit the caller's `defaults.run.working-directory`, so a `working-directory:` on the step would have done nothing and release-plz would have read the monorepo root, found no workspace naming `eql-bindings`, and exited 0 having published nothing. -- [x] Port `release-postgres-eql-image.yml` and `rebuild-docs.yml`. The image workflow kept its filename; `rebuild-docs.yml` did not survive contact. Porting its `eql-*` push-tag trigger into the root file of the same name would have added a second dead trigger to a workflow that has **never run in its life** — zero runs against `@cipherstash/stack@1.0.0`, `1.1.0` and `1.1.1`, because changesets creates those tags with `GITHUB_TOKEN` and a `GITHUB_TOKEN`-created ref does not start a `push` workflow. That is the same rule this plan already relies on for the image build. So the EQL docs-site rebuild is a job in `release.yml` that fires the webhook directly, and the `@cipherstash/stack@*` half is left as it was — a live pre-existing bug, now recorded in that file's header rather than inherited silently. +- [x] Port `release-postgres-eql-image.yml`; deliberately retire `rebuild-docs.yml`. Upstream's workflow posted to the deprecated `DOCS_WEBHOOK_URL` for an old Vercel-hosted docs site. That site is retired, so neither the EQL nor Stack release path carries the webhook forward. The versioned EQL docs bundle remains a release artifact built by `_build-eql-docs.yml`. - [ ] **Repoint trusted publishing**, only after a versioned dry run: npm for `@cipherstash/eql` (workflow `release.yml`, explicitly selecting `npm publish` under Allowed actions), and crates.io for `eql-bindings` (workflow `release-plz.yml`). -- [ ] Move the `DOCS_WEBHOOK_URL` secret and the GHCR package ownership. -- [ ] Activate the parked changesets, merge the Version Packages PR, and verify at one version V: the npm tarball, the crate, the `eql-` release with both SQL assets, the docs rebuild, and the image tags. +- [x] `GPG_PRIVATE_KEY` secret added to `cipherstash/stack` (release-plz signs its commit and tag). The workflow's GPG import is now gated on the crates.io preflight, so a run that publishes nothing does not need it. +- [x] GHCR: `cipherstash/stack` granted Write under **Manage Actions access** on `ghcr.io/cipherstash/postgres-eql`. This is an allowlist, not an exclusive binding, so the old repository can keep its access until the cutover is proven. The `org.opencontainers.image.source` label in `packages/eql/docker/Dockerfile` is what re-links the package on push and now names `cipherstash/stack` — it must merge before anything pushes. +- [x] `DOCS_WEBHOOK_URL` is deprecated with the retired docs site. Remove both the pre-existing Stack webhook workflow and the imported EQL rebuild job; it is not a cutover dependency. +- [ ] crates.io Trusted Publishing for `eql-bindings` → `cipherstash/stack` + `release-plz.yml`. **Blocked on this pipeline PR merging**: the workflow has to exist on the default branch first. +- [ ] Activate the parked changesets, merge the Version Packages PR, and verify at one version V: the npm tarball, the crate, the `eql-` release with SQL and docs assets, and the image tags. - [ ] Update the dbdev runbook for the new paths. It stays manual. ## Phase 6 — flip diff --git a/package.json b/package.json index 18b28ec98..87769b6c8 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "changeset": "changeset", "changeset:version": "pnpm run version", "changeset:publish": "pnpm run release", - "dev": "turbo dev --filter './packages/*'", + "dev": "turbo dev --filter './packages/**'", "clean": "rimraf --glob **/.next **/.turbo **/dist **/node_modules", "code:fix": "biome check --write", "code:check": "biome check", diff --git a/packages/eql/DEVELOPMENT.md b/packages/eql/DEVELOPMENT.md index 910fea132..60a65d7e4 100644 --- a/packages/eql/DEVELOPMENT.md +++ b/packages/eql/DEVELOPMENT.md @@ -421,27 +421,12 @@ Changesets assembles `CHANGELOG.md` and computes the version from these files, s do not hand-edit `CHANGELOG.md`. Behaviour callers should be aware of also needs a numbered upgrade note under `docs/upgrading/`. -#### Public documentation updates +#### Public documentation artifacts -When a tag with the `eql-` prefix is pushed (for example, `eql-1.2.3`), the -workflow at `.github/workflows/rebuild-docs.yml` runs and sends a webhook to the -Vercel-hosted public docs site to trigger a rebuild. - -What happens end-to-end: - -- Release EQL builds EQL artifacts and generates API docs (HTML, XML, Markdown). - The Markdown frontmatter includes the release version. -- Rebuild Docs posts to the `DOCS_WEBHOOK_URL` secret, which Vercel uses to kick - off a fresh build of the public docs. -- The public docs site pulls the latest generated reference - (`docs/api/markdown/API.md`) and publishes it under the corresponding version. - -Manual triggers and troubleshooting: - -- You can re-run the "Rebuild Docs" workflow from the Actions tab if a build - fails downstream. -- Ensure the repository secret `DOCS_WEBHOOK_URL` is set and valid; the workflow - simply POSTs to that URL. +EQL releases generate and attach versioned API documentation artifacts (HTML, +XML, and Markdown) alongside the SQL release. The former Vercel-hosted docs site +and its `DOCS_WEBHOOK_URL` rebuild hook have been retired; publishing an EQL +release does not trigger an external documentation deployment. ### dbdev diff --git a/packages/eql/docker/Dockerfile b/packages/eql/docker/Dockerfile index 78abc7633..f9c638a9f 100644 --- a/packages/eql/docker/Dockerfile +++ b/packages/eql/docker/Dockerfile @@ -6,7 +6,7 @@ FROM postgres:${PG_VERSION} ARG PG_VERSION=17 ARG EQL_VERSION=DEV -LABEL org.opencontainers.image.source="https://github.com/cipherstash/encrypt-query-language" +LABEL org.opencontainers.image.source="https://github.com/cipherstash/stack" LABEL org.opencontainers.image.description="PostgreSQL ${PG_VERSION} with CipherStash EQL ${EQL_VERSION} pre-installed" LABEL org.opencontainers.image.licenses="MPL-2.0" LABEL com.cipherstash.eql.version="${EQL_VERSION}" diff --git a/packages/eql/packages/eql/scripts/verify-release-assets.mjs b/packages/eql/packages/eql/scripts/verify-release-assets.mjs index cc49d8d40..44b28150d 100644 --- a/packages/eql/packages/eql/scripts/verify-release-assets.mjs +++ b/packages/eql/packages/eql/scripts/verify-release-assets.mjs @@ -13,10 +13,16 @@ const pkg = JSON.parse( readFileSync(resolve(packageRoot, 'package.json'), 'utf8'), ) +// mise finds config from the cwd and its parents, and there is no mise config +// at the repository root — the EQL subtree carries its own. A bare `mise run` +// from where these messages are read fails with a TRUST error that reads like +// a broken toolchain, so the `cd` is part of the instruction, not decoration. +const REPAIR = `(cd packages/eql && mise run release:prepare_bindings_assets --version ${pkg.version})` + const manifestPath = resolve(packageRoot, 'sql/release-manifest.json') if (!existsSync(manifestPath)) { console.error( - `refusing to publish: ${manifestPath} is missing — run 'mise run release:prepare_bindings_assets --version ${pkg.version}' first`, + `refusing to publish: ${manifestPath} is missing — run '${REPAIR}' first`, ) process.exit(1) } @@ -24,7 +30,7 @@ if (!existsSync(manifestPath)) { const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) if (manifest.eqlVersion !== pkg.version) { console.error( - `refusing to publish: sql/release-manifest.json eqlVersion ('${manifest.eqlVersion}') does not match package.json version ('${pkg.version}') — the bundled SQL was not prepared for this release. Run 'mise run release:prepare_bindings_assets --version ${pkg.version}' and rebuild.`, + `refusing to publish: sql/release-manifest.json eqlVersion ('${manifest.eqlVersion}') does not match package.json version ('${pkg.version}') — the bundled SQL was not prepared for this release. Run '${REPAIR}' and rebuild.`, ) process.exit(1) } @@ -33,9 +39,21 @@ const installSql = readFileSync( resolve(packageRoot, 'sql/cipherstash-encrypt.sql'), 'utf8', ) -if (!installSql.includes('eql_v3')) { +// THE STAMP, not the schema name. This read `installSql.includes('eql_v3')` +// until it was found unable to fire: `eql_v3` appears ~23,000 times in every +// bundle, and a DEV build is no exception — `tasks/build.sh` substitutes only +// `$RELEASE_VERSION`, which lands in this one COMMENT. So the check passed on +// any input at all, including the placeholder it names. +// +// The manifest comparison above cannot cover this either. Both files are +// regenerated together by `release:prepare_bindings_assets`, so a bundle built +// under the wrong version arrives with a manifest that agrees with it. The +// COMMENT is the only record of which build produced these bytes. Same +// reasoning, and the same predicate, as the guard in `release-plz.yml`. +if (!installSql.includes(`COMMENT ON SCHEMA eql_v3 IS '${pkg.version}'`)) { + const stamped = installSql.match(/COMMENT ON SCHEMA eql_v3 IS '[^']*'/)?.[0] console.error( - 'refusing to publish: sql/cipherstash-encrypt.sql looks like the DEV placeholder — the bundled SQL was not prepared for this release.', + `refusing to publish: sql/cipherstash-encrypt.sql was not built for ${pkg.version} — it carries ${stamped ?? 'no version stamp'}. Run '${REPAIR}' and rebuild.`, ) process.exit(1) } diff --git a/packages/eql/release-plz.toml b/packages/eql/release-plz.toml index 19c9d534f..a2fa9426c 100644 --- a/packages/eql/release-plz.toml +++ b/packages/eql/release-plz.toml @@ -20,9 +20,14 @@ # The crate is tagged `eql-bindings-v` (release-plz's default workspace # tag). That tag shape is part of the release identity, not a default nobody # chose: it deliberately does NOT match the `eql-` SQL-release -# automation, so a crate release never fires an SQL build, a Docker image, or -# the docs-site rebuild. `/.github/workflows/rebuild-docs.yml` -# carries the matching `!eql-bindings-*` exclusion. +# automation, so a crate release never fires an SQL build or a Docker image. +# +# In THIS repository that holds for a stronger reason than upstream's, and the +# difference matters if you go looking: upstream also had a deprecated docs-site +# webhook triggered by `eql-*` tags. That workflow was deliberately not ported. +# The SQL and docs artifact builds are jobs inside `release.yml`, and the image +# is dispatched from it by `gh workflow run`, so a tag of any shape reaches none +# of them. [workspace] changelog_config = "cliff.toml" diff --git a/scripts/__tests__/cargo-lock-freshness.test.mjs b/scripts/__tests__/cargo-lock-freshness.test.mjs index a417b0d9c..f37983d3f 100644 --- a/scripts/__tests__/cargo-lock-freshness.test.mjs +++ b/scripts/__tests__/cargo-lock-freshness.test.mjs @@ -43,6 +43,7 @@ import { REPO_ROOT } from './lib/repo-root.mjs' /** Same skip set, and the same reasons, as `lint-no-eql-registry-pins.mjs`. */ const SKIP_DIRS = new Set([ '.git', + '.claude', 'node_modules', 'target', 'dist', diff --git a/scripts/__tests__/cargo-publish-opt-out.test.mjs b/scripts/__tests__/cargo-publish-opt-out.test.mjs index fccf62389..7df256be5 100644 --- a/scripts/__tests__/cargo-publish-opt-out.test.mjs +++ b/scripts/__tests__/cargo-publish-opt-out.test.mjs @@ -4,29 +4,53 @@ import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' /** - * Every crate in the nested Cargo workspace must opt out of crates.io unless it - * is deliberately allowlisted below. + * Every crate in BOTH nested Cargo workspaces must opt out of crates.io unless + * it is deliberately allowlisted below. * - * A crate with no `publish` key is publishable BY DEFAULT, and `protect-ffi` - * carries none: it has never been on crates.io (verified against the registry - * API), it is a cdylib compiled into `index.node` and shipped inside the six - * `@cipherstash/protect-ffi-` npm packages, and it has no - * Rust-consumer identity at all. Nothing today would publish it — but this repo - * is about to grow a crates.io publisher (`eql-bindings`, via release-plz, when - * `cipherstash/encrypt-query-language` is absorbed), and release-plz publishes - * every workspace member that has not opted out. The convention EQL already - * uses, and which this workspace inherits with that import, is exactly one - * publishable crate with every other member explicitly `publish = false`, so - * release-plz needs no per-package configuration. + * A crate with no `publish` key is publishable BY DEFAULT, and release-plz + * publishes every workspace member that has not opted out. The convention is + * exactly one publishable crate per workspace with every other member + * explicitly `publish = false`, so release-plz needs no per-package + * configuration. * - * The list below is the audit surface: adding a name to it means "a future + * ## Why both, and why that took a second pass + * + * This checked `packages/protect-ffi` ONLY, on the reasoning — written in this + * header — that the repo "is about to grow a crates.io publisher + * (`eql-bindings`, via release-plz, when `cipherstash/encrypt-query-language` + * is absorbed)". The absorption happened. The publisher landed as + * `.github/workflows/release-plz.yml`, pointed by `manifest_path` at + * `packages/eql/Cargo.toml` — and the workspace it publishes from was the one + * workspace this file did not read. The check was strictest exactly where + * nothing could publish and absent where something can. + * + * `packages/protect-ffi/crates/protect-ffi` carries no `publish` key and is + * nonetheless correct: it has never been on crates.io (verified against the + * registry API), it is a cdylib compiled into `index.node` and shipped inside + * the six `@cipherstash/protect-ffi-` npm packages, and nothing + * publishes that workspace. It stays un-allowlisted because allowlisting means + * "release-plz will publish this", which is false for it. + * + * Each workspace's list below is the audit surface: adding a name means "a * release-plz run will publish this crate to crates.io". */ -const WORKSPACE = join(REPO_ROOT, 'packages/protect-ffi') - -/** Crates deliberately published to crates.io. Adding a name here is a decision. */ -const PUBLISHABLE = new Set([]) +const WORKSPACES = [ + { + // The crates.io publisher's target. `release-plz.yml` passes + // `manifest_path: packages/eql/Cargo.toml`. + root: 'packages/eql', + publishable: new Set(['crates/eql-bindings']), + // Pinned so an unreadable `members` list fails loudly rather than yielding + // an empty expansion that passes. + expects: 'crates/eql-bindings', + }, + { + root: 'packages/protect-ffi', + publishable: new Set(), + expects: 'crates/protect-ffi', + }, +] /** * The workspace's members, expanded from its own `[workspace] members` list. @@ -38,7 +62,7 @@ const PUBLISHABLE = new Set([]) * expansion found something, so a members list this parser cannot read fails * loudly instead of yielding an empty set that passes. */ -function workspaceMembers() { +function workspaceMembers(WORKSPACE) { const manifest = readFileSync(join(WORKSPACE, 'Cargo.toml'), 'utf8') const block = /^members\s*=\s*\[([^\]]*)\]/m.exec(manifest)?.[1] ?? '' return [...block.matchAll(/"([^"]+)"/g)] @@ -54,20 +78,34 @@ function workspaceMembers() { .sort() } -describe('cargo publish opt-out', () => { - const members = workspaceMembers() +describe.each(WORKSPACES)('cargo publish opt-out ($root)', ({ + root, + publishable, + expects, +}) => { + const workspace = join(REPO_ROOT, root) + const members = workspaceMembers(workspace) // The guard on the scan: a discovery test that enumerates nothing passes // while checking nothing. it('finds the workspace members it means to check', () => { - expect(members).toContain('crates/protect-ffi') + expect(members).toContain(expects) + }) + + // The guard on the allowlist: an entry naming a member that no longer + // exists is an exemption excusing nothing, and it would go on reading as a + // deliberate decision. + it('allowlists only real members', () => { + expect([...publishable].filter((name) => !members.includes(name))).toEqual( + [], + ) }) for (const member of members) { it(`${member} declares publish = false unless allowlisted`, () => { - if (PUBLISHABLE.has(member)) return + if (publishable.has(member)) return const manifest = readFileSync( - join(WORKSPACE, member, 'Cargo.toml'), + join(workspace, member, 'Cargo.toml'), 'utf8', ) expect(manifest).toMatch(/^publish = false$/m) diff --git a/scripts/__tests__/eql-pipeline-armed.test.mjs b/scripts/__tests__/eql-pipeline-armed.test.mjs index 5657091ed..0f320bf6a 100644 --- a/scripts/__tests__/eql-pipeline-armed.test.mjs +++ b/scripts/__tests__/eql-pipeline-armed.test.mjs @@ -34,7 +34,6 @@ const GATED_JOBS = [ '.github/workflows/release-postgres-eql-image.yml / build-images', '.github/workflows/release-postgres-eql-image.yml / build-sql', '.github/workflows/release.yml / eql-docs', - '.github/workflows/release.yml / eql-docs-rebuild', '.github/workflows/release.yml / eql-image', '.github/workflows/release.yml / eql-sql', '.github/workflows/release.yml / prerelease-eql-crate', diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 593f96560..ee1ea0311 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -1371,7 +1371,10 @@ describe('the guard fails when a workflow stops calling a cargo check', () => { * release-plz.yml -> .github/workflows/release-plz.yml, name * unchanged (crates.io binds to it) * release-postgres-eql-image.yml -> same filename at the root - * rebuild-docs.yml -> merged into the root file of that name + * rebuild-docs.yml -> DROPPED: it posted to the deprecated + * DOCS_WEBHOOK_URL for the retired docs + * site. Versioned docs artifacts are still + * built by _build-eql-docs.yml * lint-release.yml -> merged into the root file of that name * .github/actionlint.yaml -> DROPPED: the root copy already carries the * `blacksmith-16vcpu-ubuntu-2204` label @@ -1392,7 +1395,8 @@ describe('the guard fails when a workflow stops calling a cargo check', () => { * have installed a confidently wrong * document at the root. Its still-true half * — the two release tag families — is in - * AGENTS.md and in rebuild-docs.yml's header + * AGENTS.md and in packages/eql/release-plz.toml's + * header */ describe('the imported workflow directory is gone', () => { it('does not exist', () => { @@ -1423,6 +1427,19 @@ describe('the imported workflow directory is gone', () => { }) }) +describe('the retired docs site stays retired', () => { + it('has no workflow consuming the deprecated webhook secret', () => { + const consumers = workflowFiles().filter((file) => + readRepo(file).includes('DOCS_WEBHOOK_URL'), + ) + + expect( + consumers, + 'DOCS_WEBHOOK_URL belonged to the retired docs site; release workflows must publish the versioned docs artifact without invoking that webhook.', + ).toEqual([]) + }) +}) + // --------------------------------------------------------------------------- // Every job that compiles Rust restores the shared cache // --------------------------------------------------------------------------- diff --git a/scripts/__tests__/lib/package-readmes.mjs b/scripts/__tests__/lib/package-readmes.mjs new file mode 100644 index 000000000..fdfe25188 --- /dev/null +++ b/scripts/__tests__/lib/package-readmes.mjs @@ -0,0 +1,43 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { workspacePackagePatterns } from '../../release-gate.mjs' +import { REPO_ROOT } from './repo-root.mjs' + +/** + * git pathspecs selecting every workspace package's `README.md`. + * + * ## Why this is derived rather than written down + * + * Both callers used to hardcode `:(glob)packages/*/README.md`. `:(glob)` stops + * `*` at a path separator — which is what makes `lib/*.ts` behave — and this + * repo has TWO package roots nested deeper than one level: + * `packages/protect-ffi/platforms/*` and `packages/eql/packages/*`. So the + * hardcoded spec selected `packages/eql/README.md`, the 15 KB subtree root that + * ships in no tarball, and never `packages/eql/packages/eql/README.md`, the + * 518-byte file listed in that package's `files`. Same for the six per-platform + * FFI packages. Seven published READMEs were unscanned and one unpublished one + * was standing in for them, which is worse than scanning nothing: the guards + * reported green over a set that did not contain what they were guarding. + * + * `pnpm-workspace.yaml` is the only place that knows how deep a package can be, + * and it has to be edited to add one — so deriving from it means a third nested + * root is covered the day it lands, rather than the day someone remembers this + * file. Same reasoning, and the same parser, as `workspaceManifests()`. + * + * Narrowed to `packages/` deliberately: `examples/*` and `e2e` are workspace + * members too, but they are private and their READMEs are not shipped to + * anyone. The callers are guards on SHIPPED text. + */ +export function packageReadmePathspecs() { + const patterns = workspacePackagePatterns( + readFileSync(join(REPO_ROOT, 'pnpm-workspace.yaml'), 'utf8'), + ).filter((pattern) => pattern.startsWith('packages/')) + + if (patterns.length === 0) { + throw new Error( + 'pnpm-workspace.yaml lists no `packages/` patterns — either the layout moved or this derivation broke. Failing rather than returning an empty pathspec list, which `git ls-files` would answer with the whole tree.', + ) + } + + return patterns.map((pattern) => `:(glob)${pattern}/README.md`) +} diff --git a/scripts/__tests__/no-removed-drizzle-surface.test.mjs b/scripts/__tests__/no-removed-drizzle-surface.test.mjs index 1ee0f063c..b271c58f9 100644 --- a/scripts/__tests__/no-removed-drizzle-surface.test.mjs +++ b/scripts/__tests__/no-removed-drizzle-surface.test.mjs @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' +import { packageReadmePathspecs } from './lib/package-readmes.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' /** @@ -28,7 +29,9 @@ const REMOVED = [ // wildmatch crosses `/` and `lib/*.ts` sweeps in `lib/__tests__/*.test.ts`. const SHIPPED_GLOBS = [ ':(glob)skills/*/SKILL.md', - ':(glob)packages/*/README.md', + // Derived, not written down: two package roots sit deeper than one level and + // `:(glob)` does not cross `/`. See `lib/package-readmes.mjs`. + ...packageReadmePathspecs(), 'README.md', 'AGENTS.md', // `stash init` writes these strings into the user's project as real source. @@ -53,6 +56,14 @@ describe('removed stack-drizzle surface is absent from shipped files', () => { expect(files.length).toBeGreaterThan(5) expect(files).toContain('skills/stash-drizzle/SKILL.md') expect(files).toContain('packages/stack/README.md') + // The two roots nested deeper than `packages/*`. Pinned by name because + // they are what the hardcoded `:(glob)packages/*/README.md` silently + // missed: `:(glob)` does not cross `/`, so it selected the EQL subtree + // ROOT's README — which ships in no tarball — instead of the package's. + expect(files).toContain('packages/eql/packages/eql/README.md') + expect(files).toContain( + 'packages/protect-ffi/platforms/linux-x64-gnu/README.md', + ) }) it.each(files)('%s', (file) => { diff --git a/scripts/__tests__/no-removed-eql-version-flag.test.mjs b/scripts/__tests__/no-removed-eql-version-flag.test.mjs index d2aea7e6a..d11016a38 100644 --- a/scripts/__tests__/no-removed-eql-version-flag.test.mjs +++ b/scripts/__tests__/no-removed-eql-version-flag.test.mjs @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' +import { packageReadmePathspecs } from './lib/package-readmes.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' /** Tracked executable examples that describe the current public surface. */ @@ -12,7 +13,9 @@ function publicCommandDocs() { 'ls-files', '-z', ':(glob)skills/*/SKILL.md', - ':(glob)packages/*/README.md', + // Derived, not written down: two package roots sit deeper than one level + // and `:(glob)` does not cross `/`. See `lib/package-readmes.mjs`. + ...packageReadmePathspecs(), 'docs/reference/supabase-sdk.md', ], { @@ -77,6 +80,14 @@ describe('public eql install examples use the current CLI', () => { expect(files).toContain('skills/stash-supabase/SKILL.md') expect(files).toContain('packages/stack/README.md') expect(files).toContain('docs/reference/supabase-sdk.md') + // The two roots nested deeper than `packages/*`. Pinned by name because + // they are what the hardcoded `:(glob)packages/*/README.md` silently + // missed: `:(glob)` does not cross `/`, so it selected the EQL subtree + // ROOT's README — which ships in no tarball — instead of the package's. + expect(files).toContain('packages/eql/packages/eql/README.md') + expect(files).toContain( + 'packages/protect-ffi/platforms/linux-x64-gnu/README.md', + ) }) it.each(files)('%s', (file) => { diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index d9557fb18..e4dff0592 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -97,6 +97,20 @@ const EXPECTED_PARITY_WORKFLOWS = [ * reason has to explain why the drift it guards against is not possible. */ const EXPECTED_ASYMMETRIES = new Map([ + [ + '.github/workflows/release-plz.yml', + // There is no `pull_request:` trigger at all, and there must not be: this + // workflow PUBLISHES the `eql-bindings` crate to crates.io over OIDC. A + // pull_request copy would be a publishing job reachable from a fork branch. + // So there is no second list to drift from. The direction that would hurt + // here is not a narrow filter but a MISSING run — a release that moves the + // crate and never triggers — and that is unreachable by construction: + // `changeset version` runs scripts/sync-lockstep-versions.mjs, which + // rewrites packages/eql/crates/eql-bindings/Cargo.toml and the SQL assets + // beside it, so every version-moving commit touches `packages/eql/**`. + // The list itself is kept honest by eql-workflow-filters.test.mjs. + 'push is the only trigger; a pull_request copy would make a crates.io publisher reachable from a fork', + ], [ '.github/workflows/tests-rust.yml', // `push:` is `branches: [main]` with no `paths:`, so every push to main runs diff --git a/scripts/lint-no-eql-registry-pins.mjs b/scripts/lint-no-eql-registry-pins.mjs index c82749013..aacc6a669 100644 --- a/scripts/lint-no-eql-registry-pins.mjs +++ b/scripts/lint-no-eql-registry-pins.mjs @@ -104,9 +104,19 @@ const WORKSPACE_FILE = 'pnpm-workspace.yaml' * `target/package`, and every transitive dependency under `node_modules` — * which name their own dependencies by registry version, correctly. Scanning * them turns this linter into a permanent false alarm. + * + * `.claude` is the same argument one tool along. Claude Code puts agent + * worktrees under `.claude/worktrees/`, and a worktree is a FULL checkout of + * some other commit — including ones predating the in-tree flip, which still + * pin `@cipherstash/eql` by registry version and were correct when they were + * written. Descending into them reported 31 offenders and exited 1 on a clean + * tree. It is excluded through `.git/info/exclude`, which is local and travels + * to no other checkout, so `git check-ignore` is not a substitute for naming + * it here. */ const SKIP_DIRS = new Set([ '.git', + '.claude', 'node_modules', 'target', 'dist', From cbb053aaf62852b83a8091e152ffaa4c77f2a16a Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 25 Aug 2026 12:42:44 +1000 Subject: [PATCH 4/5] fix(eql): force versioned SQL release builds --- .github/workflows/_build-eql-sql.yml | 4 +++- .../eql-sql-asset-freshness.test.mjs | 21 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build-eql-sql.yml b/.github/workflows/_build-eql-sql.yml index d426bf9b8..f27115dd1 100644 --- a/.github/workflows/_build-eql-sql.yml +++ b/.github/workflows/_build-eql-sql.yml @@ -80,9 +80,11 @@ jobs: - name: Build EQL release # Strip `eql-` so eql_v3.version() reports bare semver. An empty TAG # falls through to tasks/build.sh's `${usage_version:-DEV}` default. + # `--version` is not one of build.sh's declared mise sources, so force + # regeneration when unchanged SQL is being stamped for a new release. env: TAG: ${{ inputs.tag }} - run: mise run build --version "${TAG#eql-}" + run: mise run --force build --version "${TAG#eql-}" # Artifact and release paths are workspace-root relative. - name: Upload EQL artifacts diff --git a/scripts/__tests__/eql-sql-asset-freshness.test.mjs b/scripts/__tests__/eql-sql-asset-freshness.test.mjs index 62f188828..f16f2911b 100644 --- a/scripts/__tests__/eql-sql-asset-freshness.test.mjs +++ b/scripts/__tests__/eql-sql-asset-freshness.test.mjs @@ -1,5 +1,11 @@ import { createHash } from 'node:crypto' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' @@ -73,6 +79,19 @@ const { checked, skew } = eqlLockstepSkew({ version: VERSION, }) +describe('the release SQL build stamps the requested version', () => { + it('forces mise to rebuild when only --version changed', () => { + const workflow = readFileSync( + join(REPO_ROOT, '.github/workflows/_build-eql-sql.yml'), + 'utf8', + ) + + expect(workflow).toContain( + 'mise run --force build --version "${TAG#eql-}"', + ) + }) +}) + describe('the EQL lockstep artefacts all declare one version', () => { it('reads a version to compare against', () => { // The floor. `readEqlVersion` throws rather than returning undefined, but a From 37c32967ba73a69c10db604445c663bab92f1807 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 25 Aug 2026 13:24:46 +1000 Subject: [PATCH 5/5] fix(eql): satisfy Biome in SQL freshness guard --- scripts/__tests__/eql-sql-asset-freshness.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/__tests__/eql-sql-asset-freshness.test.mjs b/scripts/__tests__/eql-sql-asset-freshness.test.mjs index f16f2911b..2732b4562 100644 --- a/scripts/__tests__/eql-sql-asset-freshness.test.mjs +++ b/scripts/__tests__/eql-sql-asset-freshness.test.mjs @@ -86,8 +86,9 @@ describe('the release SQL build stamps the requested version', () => { 'utf8', ) + const tagExpansion = '${' + 'TAG#eql-}' expect(workflow).toContain( - 'mise run --force build --version "${TAG#eql-}"', + `mise run --force build --version "${tagExpansion}"`, ) }) })