diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..dfd82a6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,99 @@ +# Reusable build + smoke-test workflow. Called by ci.yml (PRs and main) and +# publish.yml (tag push). Each job uses the same Makefile targets used in the +# local flow ('make one TARGET=... VERSION=...') and runs the deterministic +# MCP smoke test on a runner that can execute the embedded binary. + +name: build + +on: + workflow_call: + inputs: + version: + description: stackql release version without the leading v (e.g. 0.10.500) + required: true + type: string + +permissions: + contents: read + +jobs: + bundle: + name: ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + include: + - target: linux-x64 + runner: ubuntu-latest + # arm64 runners are free for public repos. If this repo ever goes + # private, swap to a paid arm runner or build this target on + # ubuntu-latest and drop the smoke test (packaging itself has no + # arch dependency - only executing the binary does). + - target: linux-arm64 + runner: ubuntu-24.04-arm + # macos runners have pkgutil, node, and shasum preinstalled, which + # is everything the darwin slice needs. + - target: darwin-universal + runner: macos-latest + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v6 + + - name: Build bundle + run: make one TARGET=${{ matrix.target }} VERSION=${{ inputs.version }} + + - name: Smoke test (deterministic MCP handshake) + run: python3 scripts/smoke-test.py dist/stackql-mcp-${{ matrix.target }}.mcpb + + # Soft agent check; exits 0 with a notice when GEMINI_API_KEY is unset. + - name: Agent smoke test (optional) + if: matrix.target == 'linux-x64' + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: python3 scripts/gemini-smoke.py dist/stackql-mcp-linux-x64.mcpb + + - name: Upload bundle artifact + uses: actions/upload-artifact@v7 + with: + name: stackql-mcp-${{ matrix.target }} + path: | + dist/stackql-mcp-${{ matrix.target }}.mcpb + dist/stackql-mcp-${{ matrix.target }}.mcpb.sha256 + if-no-files-found: error + + # The windows bundle has no build-time Windows dependency (bash + unzip + # extract stackql.exe from the release zip, node packs it), so it builds on + # ubuntu. Only the smoke test needs a Windows runner to execute the binary. + bundle-windows: + name: windows-x64 (build) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build bundle + run: make one TARGET=windows-x64 VERSION=${{ inputs.version }} + + - name: Upload bundle artifact + uses: actions/upload-artifact@v7 + with: + name: stackql-mcp-windows-x64 + path: | + dist/stackql-mcp-windows-x64.mcpb + dist/stackql-mcp-windows-x64.mcpb.sha256 + if-no-files-found: error + + smoke-windows: + name: windows-x64 (smoke) + needs: bundle-windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v6 + + - name: Download bundle artifact + uses: actions/download-artifact@v8 + with: + name: stackql-mcp-windows-x64 + path: dist + + - name: Smoke test (deterministic MCP handshake) + run: python scripts/smoke-test.py dist/stackql-mcp-windows-x64.mcpb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cbe1b46 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +# PR / main CI: build and smoke-test all four bundles for the stackql release +# pinned in release.yaml. Nothing is published from this workflow - publishing +# happens in publish.yml when a matching v-tag is pushed. + +name: ci + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + version: + name: read release.yaml + runs-on: ubuntu-latest + outputs: + version: ${{ steps.cfg.outputs.version }} + steps: + - uses: actions/checkout@v6 + + - name: Read pinned stackql release + id: cfg + run: | + tag="$(sed -n 's/^stackql_release:[[:space:]]*//p' release.yaml | tr -d ' \r')" + if [ -z "$tag" ]; then + echo "::error::stackql_release not set in release.yaml" + exit 1 + fi + echo "pinned release: $tag" + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" + + build: + needs: version + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.version.outputs.version }} + secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..8680de0 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,69 @@ +# Tag-push release: verify the tag matches release.yaml, rebuild and +# smoke-test all four bundles, then upload them (plus .sha256 companions) to +# the matching stackql/stackql GitHub release. +# +# Requires one repo secret: +# STACKQL_RELEASE_TOKEN - fine-grained PAT (or GitHub App token) with +# contents:write on stackql/stackql. The default GITHUB_TOKEN is scoped to +# this repo and cannot upload assets cross-repo. + +name: publish + +on: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + verify-tag: + name: tag matches release.yaml + runs-on: ubuntu-latest + outputs: + version: ${{ steps.cfg.outputs.version }} + steps: + - uses: actions/checkout@v6 + + - name: Compare tag against pinned release + id: cfg + run: | + cfg="$(sed -n 's/^stackql_release:[[:space:]]*//p' release.yaml | tr -d ' \r')" + tag="$GITHUB_REF_NAME" + if [ "$tag" != "$cfg" ]; then + echo "::error::tag '$tag' does not match stackql_release '$cfg' in release.yaml" + exit 1 + fi + echo "publishing for $tag" + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" + + build: + needs: verify-tag + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.verify-tag.outputs.version }} + secrets: inherit + + publish: + name: upload to stackql/stackql release + needs: [verify-tag, build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Gather bundle artifacts + uses: actions/download-artifact@v8 + with: + pattern: stackql-mcp-* + path: dist + merge-multiple: true + + - name: List artefacts + run: ls -l dist/ + + # 'make publish' uses gh release upload --clobber, so re-running this + # workflow (or overlapping with a manual local publish) is safe. + - name: Upload to release + env: + GH_TOKEN: ${{ secrets.STACKQL_RELEASE_TOKEN }} + run: make publish VERSION=${{ needs.verify-tag.outputs.version }} diff --git a/CLAUDE.md b/CLAUDE.md index 87a4e1f..ff9a2d4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,11 +14,14 @@ The server packed into each bundle is the `stackql` binary itself, launched as ` A [Makefile](Makefile) wraps `scripts/package.sh` for the common flows. The script is still the source of truth; `make` is convenience. +`VERSION` defaults to the `stackql_release` value pinned in [release.yaml](release.yaml) (leading `v` stripped), so plain `make all` builds the pinned release. Passing `VERSION=X.Y.Z` overrides it. `release.yaml` is also what CI builds on PRs and what a pushed tag must match to publish (see "Release flow" below). + One-shot from a clean checkout - downloads the release artefacts from `https://github.com/stackql/stackql/releases/download/v/...` into `bin/`, then builds every available bundle: ```bash make all VERSION=X.Y.Z # 'make VERSION=X.Y.Z' is equivalent ('all' is the default target) +# 'make' alone uses the version pinned in release.yaml ``` Just download (skip packaging): @@ -76,11 +79,21 @@ Show what is currently in the drop-zone: make list ``` -## Release flow (fully local, no CI) +## Release flow + +The primary flow is GitHub Actions; the two-machine local flow below remains a supported fallback. The darwin target needs `pkgutil`, so CI builds it on a `macos-latest` runner. + +### CI flow (GitHub Actions) + +Three workflows in [.github/workflows/](.github/workflows/): + +- **[build.yml](.github/workflows/build.yml)** - reusable. Builds each bundle with `make one TARGET= VERSION=` on a runner that can execute the embedded binary (`ubuntu-latest`, `ubuntu-24.04-arm`, `macos-latest`), runs `scripts/smoke-test.py` against it, and uploads `dist/` artefacts. The windows-x64 bundle builds on ubuntu (packaging has no Windows dependency) and is smoke-tested on `windows-latest` from the artifact. The Gemini agent check runs on linux-x64 only and soft-skips without `GEMINI_API_KEY`. +- **[ci.yml](.github/workflows/ci.yml)** - on PRs to main and pushes to main. Reads `stackql_release` from [release.yaml](release.yaml) and calls build.yml. Nothing is published. +- **[publish.yml](.github/workflows/publish.yml)** - on pushing a `v*` tag. Fails fast if the tag does not exactly match `stackql_release` in release.yaml, rebuilds and smoke-tests everything, then runs `make publish` to upload all bundles + `.sha256` files to the matching `stackql/stackql` release. Requires the `STACKQL_RELEASE_TOKEN` repo secret (fine-grained PAT with `contents:write` on `stackql/stackql` - the default `GITHUB_TOKEN` cannot upload cross-repo). -There is no GitHub Actions workflow. Releases are produced by running `make` on two machines, because the darwin target needs `pkgutil` which only exists on macOS. +The release sequence: upstream `stackql/stackql` release publishes the core assets -> raise a PR here bumping `stackql_release` in release.yaml (CI proves the bundles build and pass smoke tests against the real release assets) -> merge -> push the matching tag (e.g. `v0.10.500`) -> publish.yml attaches the `.mcpb` assets to the upstream release. -### The two-machine flow +### Fallback: the two-machine local flow **Machine A (your workstation, any OS with bash + node + unzip):** diff --git a/Makefile b/Makefile index 3f637c0..88c5829 100644 --- a/Makefile +++ b/Makefile @@ -32,10 +32,15 @@ SHELL := bash .SHELLFLAGS := -eu -o pipefail -c -VERSION ?= +# VERSION defaults to the stackql_release pinned in release.yaml (leading v +# stripped), so plain 'make all' builds the pinned release. Override with +# make VERSION=X.Y.Z as before. +VERSION ?= $(shell sed -n 's/^stackql_release:[[:space:]]*v\{0,1\}//p' release.yaml 2>/dev/null | tr -d ' \r') BIN_DIR ?= bin DIST_DIR ?= dist -PACKAGE := scripts/package.sh +# Invoked via 'bash' so a lost executable bit (easy when committing from +# Windows) cannot break the build. +PACKAGE := bash scripts/package.sh RELEASE_BASE := https://github.com/stackql/stackql/releases/download ASSETS := stackql_linux_amd64.zip \ @@ -74,7 +79,8 @@ help: check-version: @if [ -z "$(VERSION)" ]; then \ - echo "error: VERSION is required (e.g. make VERSION=0.10.500)" >&2; exit 2; \ + echo "error: VERSION is required (e.g. make VERSION=0.10.500)," >&2; \ + echo " or set stackql_release in release.yaml" >&2; exit 2; \ fi download: check-version @@ -136,7 +142,7 @@ signed: check-version # workstation, after the Mac slice has been published and downloaded back, or # after copying the darwin sha file across). server-json: check-version - scripts/render-server-json.sh --version $(VERSION) + bash scripts/render-server-json.sh --version $(VERSION) # Publish the rendered server.json to the Official MCP Registry. # Requires: @@ -194,7 +200,7 @@ list: @ls -1 $(BIN_DIR) 2>/dev/null | grep -v -E '^(\.gitignore|README\.md)$$' || echo "(empty)" clean: - scripts/clean.sh + bash scripts/clean.sh clean-bin: @rm -f $(addprefix $(BIN_DIR)/,$(ASSETS)) diff --git a/README.md b/README.md index 077f93d..5dce6e8 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ The end-user install story is in [docs/install.md](docs/install.md). The marketp - [What gets packaged](#what-gets-packaged) - [Layout](#layout) - [Prerequisites](#prerequisites) -- [Release runbook (the whole thing)](#release-runbook-the-whole-thing) +- [CI release flow (GitHub Actions)](#ci-release-flow-github-actions) +- [Release runbook (local fallback)](#release-runbook-local-fallback) - [Step 0 - one-time setup, per machine](#step-0---one-time-setup-per-machine) - [Step 1 - build and publish bundles (Machine A: workstation)](#step-1---build-and-publish-bundles-machine-a-workstation) - [Step 2 - build and publish darwin (Machine B: Mac)](#step-2---build-and-publish-darwin-machine-b-mac) @@ -38,6 +39,10 @@ One bundle is produced per target: ``` stackql-mcpb-packaging/ + release.yaml # pins the stackql release this repo packages + .github/workflows/build.yml # reusable: build + smoke-test all bundles + .github/workflows/ci.yml # PRs / main: build + test, no publish + .github/workflows/publish.yml # v* tag: verify, build, test, publish manifest/manifest.template.json # MCPB manifest, tokenised (__VERSION__, __BINARY_NAME__) registry/server.template.json # Official MCP Registry server.json, tokenised SHAs + VERSION scripts/package.sh # build bundles from bin/ -> dist/ @@ -71,11 +76,26 @@ For Step 3: - **`mcp-publisher`** CLI - https://github.com/modelcontextprotocol/registry/releases/latest -## Release runbook (the whole thing) +## CI release flow (GitHub Actions) -There is **no CI**. Releases are produced locally on two machines because the darwin target needs `pkgutil` (macOS-only). Each machine independently uploads what it built; `--clobber` makes order irrelevant and re-runs safe. +The primary release path. The stackql release being packaged is pinned in [release.yaml](release.yaml) as `stackql_release: vX.Y.Z`, which is the single source of truth for local `make` defaults, PR CI, and tag publishing. -Throughout, `VERSION` is the stackql release minus the leading `v`. For example, tag `v0.10.500` -> `VERSION=0.10.500`. +The sequence: + +1. **Upstream release happens** - `stackql/stackql` publishes `vX.Y.Z` with the core assets (per-arch zips and the notarised `.pkg`). +2. **PR bumps the pin** - raise a PR to main changing `stackql_release` in `release.yaml`. [ci.yml](.github/workflows/ci.yml) builds all four bundles against the real release assets and runs the deterministic smoke test on a native runner per platform (`ubuntu-latest`, `ubuntu-24.04-arm`, `windows-latest`, `macos-latest` - the darwin slice runs `pkgutil` on the macos runner). A green PR means the bundles build and the embedded binaries speak MCP. +3. **Merge to main** - nothing is published yet. +4. **Push the matching tag** - `git tag vX.Y.Z && git push origin vX.Y.Z`. [publish.yml](.github/workflows/publish.yml) fails fast if the tag does not exactly match `release.yaml`, rebuilds and re-tests everything, then uploads all `.mcpb` + `.sha256` files to the `stackql/stackql` `vX.Y.Z` release via `make publish` (idempotent `--clobber`). + +One-time setup: add a repo secret `STACKQL_RELEASE_TOKEN` - a fine-grained PAT (or GitHub App token) with `contents:write` on `stackql/stackql`. The default `GITHUB_TOKEN` cannot upload assets to another repo. Optionally add `GEMINI_API_KEY` to enable the agent smoke test on the linux-x64 job; without it that step soft-skips. + +Steps 3 and 4 of the local runbook below (MCP Registry publish and aggregator listings) are still manual after a CI publish. + +## Release runbook (local fallback) + +The pre-CI flow, kept as a supported fallback (and for the registry/listings steps CI does not cover). Releases are produced locally on two machines because the darwin target needs `pkgutil` (macOS-only). Each machine independently uploads what it built; `--clobber` makes order irrelevant and re-runs safe. + +Throughout, `VERSION` is the stackql release minus the leading `v`. For example, tag `v0.10.500` -> `VERSION=0.10.500`. If `VERSION` is omitted, `make` defaults it from `release.yaml`. ### Step 0 - one-time setup, per machine @@ -223,6 +243,7 @@ gh release download v0.10.500 --repo stackql/stackql \ | `make publish` | GitHub token with `contents:write` on `stackql/stackql` | `gh auth login` (token in gh) | Same login on both Machine A and Machine B. | | `make server-json` | all four `dist/*.sha256` files | files on disk | Step 3.1 fetches the darwin one from the release page. | | `make registry-publish` | `mcp-publisher login github` for an account on the stackql org | token in `mcp-publisher` config | Browser-flow OAuth; refresh annually. | +| CI publish (`publish.yml`) | `STACKQL_RELEASE_TOKEN` repo secret | fine-grained PAT | `contents:write` on `stackql/stackql`. Default `GITHUB_TOKEN` cannot upload cross-repo. | | Anthropic Desktop Extensions submission | privacy policy, logo, screenshots, contacts | filled into web form | See [docs/anthropic-submission.md](docs/anthropic-submission.md). | No secrets are passed via env vars in the build/publish commands themselves - tokens live in the per-tool config of `gh` and `mcp-publisher`. The one exception is `GEMINI_API_KEY` for the optional Gemini soft check. diff --git a/release.yaml b/release.yaml new file mode 100644 index 0000000..2070b0b --- /dev/null +++ b/release.yaml @@ -0,0 +1,9 @@ +# Single source of truth for which stackql release this repo packages. +# +# - Local: make targets default VERSION from this value (leading v stripped), +# so 'make all' with no VERSION builds the pinned release. +# - CI (PR to main): builds and smoke-tests all four bundles for this release. +# - CI (tag push): pushing a tag that exactly matches this value publishes the +# bundles to the matching stackql/stackql release. A tag that does not match +# fails fast in the verify-tag job. +stackql_release: v0.10.500 diff --git a/scripts/clean.sh b/scripts/clean.sh old mode 100644 new mode 100755 diff --git a/scripts/package.sh b/scripts/package.sh old mode 100644 new mode 100755 diff --git a/scripts/render-server-json.sh b/scripts/render-server-json.sh old mode 100644 new mode 100755 diff --git a/scripts/smoke-test.py b/scripts/smoke-test.py index bb6d600..9f2ad21 100644 --- a/scripts/smoke-test.py +++ b/scripts/smoke-test.py @@ -66,8 +66,8 @@ def __init__(self, proc: subprocess.Popen) -> None: def _read_loop(self) -> None: assert self.proc.stdout is not None - for line in self.proc.stdout: - line = line.strip() + for raw in self.proc.stdout: + line = raw.decode("utf-8", errors="replace").strip() if not line: continue try: @@ -86,7 +86,7 @@ def send(self, method: str, params: dict | None = None, *, id_: int | None = Non msg["id"] = id_ line = json.dumps(msg) + "\n" assert self.proc.stdin is not None - self.proc.stdin.write(line) + self.proc.stdin.write(line.encode("utf-8")) self.proc.stdin.flush() def wait(self, id_: int, timeout: float) -> dict: @@ -111,13 +111,13 @@ def run(bundle_path: Path) -> None: cmd = [str(binary), "mcp", "--mcp.server.type=stdio", f"--auth={GITHUB_AUTH}"] log(f"spawning: {' '.join(cmd)}") + # Binary pipes on purpose: text=True would translate \n to \r\n on + # Windows stdin, and the server exits silently on the stray \r. proc = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, - bufsize=1, ) try: client = JsonRpcClient(proc)