ci(release): publish multi-arch amd64+arm64 images from native runners

.github/workflows/multiarch-image.yml

@@ -0,0 +1,134 @@
+# PR gate for the release image build path: native amd64 + native arm64,
+# push-by-digest, then a throwaway multi-arch tag `pr-<n>` so arm64 hosts can
+# `docker pull` the merged manifest without cutting a release. QEMU is
+# intentionally not used (see ADR-0003).
+name: Multi-arch image build
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/multiarch-image.yml"
+      - ".github/workflows/release.yml"
+      - "Dockerfile"
+      - "go.mod"
+      - "go.sum"
+      - "cmd/**"
+      - "internal/**"
+      - "frontend/**"
+      - "scripts/entrypoint.sh"
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build:
+    name: Build ${{ matrix.platform }}
+    # Fork PRs get a read-only GITHUB_TOKEN; packages:write cannot elevate it.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            digest_name: linux-amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            digest_name: linux-arm64
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Lowercase image name
+        id: image
+        run: |
+          echo "name=${{ env.REGISTRY }}/$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,name=${{ steps.image.outputs.name }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=multiarch-${{ matrix.digest_name }}
+          cache-to: type=gha,scope=multiarch-${{ matrix.digest_name }},mode=max
+
+      - name: Export digest
+        run: |
+          mkdir -p "${{ runner.temp }}/digests"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.digest_name }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Publish multi-arch tag
+    needs: build
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Lowercase image name
+        id: image
+        run: |
+          echo "name=${{ env.REGISTRY }}/$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
+
+      - name: Tag name
+        id: tag
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "name=pr-${{ github.event.number }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "name=multiarch-test" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          IMAGE="${{ steps.image.outputs.name }}"
+          TAG="${{ steps.tag.outputs.name }}"
+          docker buildx imagetools create \
+            -t "${IMAGE}:${TAG}" \
+            $(printf "${IMAGE}@sha256:%s " *)
+          echo "Published ${IMAGE}:${TAG}"
+          docker buildx imagetools inspect "${IMAGE}:${TAG}"

.github/workflows/release.yml

@@ -15,15 +15,20 @@ on:
 
 env:
   REGISTRY: ghcr.io
+  # GHCR digests require a lowercase repository path.
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  release:
-    name: Create release
+  prepare:
+    name: Tag release
     runs-on: ubuntu-latest
     permissions:
       contents: write
-      packages: write
+    outputs:
+      tag: ${{ steps.version.outputs.tag }}
+      version: ${{ steps.version.outputs.version }}
+      image: ${{ steps.image.outputs.name }}
+      revision: ${{ steps.commit.outputs.sha }}
     steps:
       - name: Ensure running on main
         run: |
@@ -32,6 +37,11 @@ jobs:
             exit 1
           fi
 
+      - name: Lowercase image name
+        id: image
+        run: |
+          echo "name=${{ env.REGISTRY }}/$(echo '${{ env.IMAGE_NAME }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
+
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
@@ -97,13 +107,98 @@ jobs:
           git config user.email "github-actions[bot]@users.noreply.github.com"
 
       - name: Commit and tag
+        id: commit
         run: |
           git add CHANGELOG.md
           git commit -m "chore: release ${{ steps.version.outputs.tag }}"
           git tag -a "${{ steps.version.outputs.tag }}" \
             -m "Release ${{ steps.version.outputs.tag }}"
           git push origin HEAD:main
           git push origin "${{ steps.version.outputs.tag }}"
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: release-notes
+          path: release-notes.md
+
+  build:
+    name: Build ${{ matrix.platform }}
+    needs: prepare
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            digest_name: linux-amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            digest_name: linux-arm64
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ needs.prepare.outputs.tag }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          labels: |
+            org.opencontainers.image.version=${{ needs.prepare.outputs.version }}
+            org.opencontainers.image.revision=${{ needs.prepare.outputs.revision }}
+          outputs: type=image,name=${{ needs.prepare.outputs.image }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=${{ matrix.digest_name }}
+          cache-to: type=gha,scope=${{ matrix.digest_name }},mode=max
+
+      - name: Export digest
+        run: |
+          mkdir -p "${{ runner.temp }}/digests"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.digest_name }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Manifest and GitHub Release
+    needs: [prepare, build]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Download release notes
+        uses: actions/download-artifact@v4
+        with:
+          name: release-notes
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
@@ -119,26 +214,26 @@ jobs:
         id: meta
         uses: docker/metadata-action@v6
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ needs.prepare.outputs.image }}
           tags: |
-            type=semver,pattern={{version}},value=${{ steps.version.outputs.tag }}
-            type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.tag }}
+            type=semver,pattern={{version}},value=${{ needs.prepare.outputs.tag }}
+            type=semver,pattern={{major}}.{{minor}},value=${{ needs.prepare.outputs.tag }}
             type=raw,value=latest
 
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v7
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ needs.prepare.outputs.image }}@sha256:%s ' *)
 
       - name: Create GitHub Release
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # This job has no checkout, so gh has no git remote to resolve the
+          # repository from; it does not read GITHUB_REPOSITORY.
+          GH_REPO: ${{ github.repository }}
         run: |
-          gh release create "${{ steps.version.outputs.tag }}" \
-            --title "cotel ${{ steps.version.outputs.tag }}" \
+          gh release create "${{ needs.prepare.outputs.tag }}" \
+            --title "cotel ${{ needs.prepare.outputs.tag }}" \
             --notes-file release-notes.md

CHANGELOG.md

@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A failing `GET /api/v1/bash-commands` no longer renders as the "no command detail in this data" explainer. The Bash section branched on row count alone, so a request that errored looked identical to one that legitimately returned nothing, blaming Claude Code's telemetry for what was actually a server fault. Fetch failures now show the error
 
 ### Changed
+- Release images are multi-arch (`linux/amd64` + `linux/arm64`). `release.yml` builds each platform on a native hosted runner and merges a manifest list, so `docker pull ghcr.io/flopsstuff/cotel:latest` works on arm64 hosts (Pi, Apple Silicon) without a local `docker compose build` before restore. QEMU-emulated arm64 on a single amd64 runner was measured and rejected; native `ubuntu-24.04-arm` finished the same Dockerfile in ~47s ([ADR-0003](docs/decisions/0003-release-policy.md), [docs/operations/duckdb-snapshots.md](docs/operations/duckdb-snapshots.md))
 - The bundled `cloudflared` is **2026.9.3** in place of 2024.11.1, still an explicit pin rather than `latest` so rebuilding an old commit produces the binary it originally shipped. The two years in between hold one change that touches cotel: 2026.4.0 flipped the `--edge-ip-version` default from `4` to `auto`, which connects over whichever address family the system resolver answers with first and falls back to the other only after a connection has already failed - so a container whose resolver answers `AAAA` first but has no working IPv6 egress spends its first attempts failing, and then sticks with the fallback for ten minutes at a time. Token mode therefore sets `TUNNEL_EDGE_IP_VERSION=4` unless the operator sets it, holding the behaviour the old version had; local-config mode is deliberately left alone, because that env var silently outranks an `edge-ip-version` in the operator's own `config.yml`. The other new arrival is visible rather than behavioural: cloudflared now probes DNS, QUIC, HTTP/2 and the Cloudflare API at startup and logs a `CONNECTIVITY PRE-CHECKS` table, which is kept on - it runs concurrently with startup and does not gate it, a `FAIL` row does not stop the tunnel, and it is the fastest way to separate a blocked UDP path from a bad token, which is exactly the diagnosis the deploy health gate cannot make because it probes cotel's `/healthz` and not the tunnel. The bump does **not** address the token-in-logs leak and was kept out of that fix for the same reason the fix was kept out of this bump: measured on 2026.9.3, a token left in the environment is still printed in full by the startup env dump. `--token-file`, which the new version adds, is not adopted either - the token stays readable through `docker inspect` and `/proc/1/environ` regardless, so it would close no access path in exchange for a tmpfs mount and a secret on disk ([docs/operations/cloudflare-tunnel-remote.md](docs/operations/cloudflare-tunnel-remote.md))
 - The DuckDB driver is `github.com/duckdb/duckdb-go/v2`, which carries engine **1.5.6** in place of 1.1.3. The reason is not the version: `github.com/marcboeker/go-duckdb` is deprecated by its own author — the first line of its `go.mod` reads *"This module has moved to github.com/duckdb/duckdb-go"* and its last tag is a year old — so cotel was pinned to an address that is no longer the dependency and has no upstream to receive a bug report. This is **not** a fix for the ART-metadata corruption that took production down: upstream [duckdb#25360](https://github.com/duckdb/duckdb/issues/25360) reports the same class of fault still reproducing on 1.5.5, and what the newer engine changes is that it raises a Go error instead of `abort()`ing inside cgo. The on-disk format was the thing that needed measuring, since a newer engine writing a file the previous image cannot read would make the upgrade unrollbackable with production's only copy of the data inside it — and it does not: an existing file is never converted by ordinary open-and-write, not even when `storage_compatibility_version=v1.5.6` is asked for explicitly, and 1.1.3 reads a file 1.5.6 has written to, replays a WAL 1.5.6 left, and was confirmed doing so as the actual `ghcr.io/flopsstuff/cotel:latest` image. The door closes only for a *newly created* file at an explicit opt-in, so `storage.Open` now pins `storage_compatibility_version=v0.10.2` in the DSN and a test reads the format number straight out of the file header to hold it at 64 — the pin is a no-op against today's engine, which is the point: a default is behaviour, not a contract, and a future minor could raise it and leave no diff of ours to point at. Two further consequences: Go **1.24.0** is now the build floor (`go.mod`, the builder image and CI move together), and the runtime image grows 213 MB → 294 MB for the bigger static engine library — against which ICU is now linked in rather than fetched over the network on every cold start. A DuckDB `JSON` column no longer scans into a Go `*string` (v2 hands back a decoded `map[string]interface{}`), so `ExportSpans` and the session-detail query cast in SQL; any new query reading `attributes` or `resource_attrs` into a string needs the same `::VARCHAR` ([ADR-0018](docs/decisions/0018-duckdb-go-v2-driver.md))
 - The chart palette is re-derived — eight of its ten values change, `--color-chart-1` is unchanged in both schemes because it is `--color-accent`. The old palette broke two things at once. Its dark values were the light values lightened, which reverses loudness when the surface flips: the quietest series in light was the loudest in dark, so the Overview's cost line shouted over the spans area it annotates for any reader on a dark theme, and did not for anyone on a light one. And it was not the colour-independent palette it was documented to be — `chart-1` and `chart-2` measured ΔE 0.4 light / 0.3 dark under deuteranopia, the same colour to roughly one man in twelve, and the palette is handed out in order, so that was the pair every two-series chart in the product drew. The new values hold both rules with margin: identical salience rank in both schemes, and every one of the ten pairs ≥ 8.6 ΔE under protanopia, deuteranopia and tritanopia in both schemes, measured with Machado 2009 at severity 1.0 in OKLab. Each token keeps its hue to within 3° — the palette still reads blue, violet, green, amber, pink — and only lightness and chroma moved; the contrast floor against the surface rises from 3.19:1 to 4.08:1, above the 3:1 WCAG 1.4.11 asks of a 2 px stroke. `--color-chart-4` also stops being byte-identical to `--color-warning`, which it was in both schemes: the cost line was literally the warning colour. The rules, the ruler and the full pairwise measurement are in [tokens.md](docs/design/tokens.md)

docs/decisions/0003-release-policy.md

@@ -31,12 +31,16 @@ Releases are created **only** by manually triggering `.github/workflows/release.
 2. Accepts a `bump` input (`minor` default, `patch`, `major`).
 3. Updates `CHANGELOG.md` and commits it.
 4. Creates an annotated git tag and pushes both the commit and the tag.
-5. Builds and pushes the Docker image to GHCR with `vMAJOR.MINOR.PATCH`, `vMAJOR.MINOR`, and `latest` tags.
+5. Builds and pushes a **multi-arch** Docker image to GHCR (`linux/amd64` and `linux/arm64`) with `vMAJOR.MINOR.PATCH`, `vMAJOR.MINOR`, and `latest` tags.
 6. Creates a GitHub Release with the extracted changelog section as release notes.
 
+### Image platforms
+
+The published image must be pullable on both amd64 hosts and arm64 hosts (Apple Silicon production, Raspberry Pi restore drills). A single `platforms: linux/amd64,linux/arm64` build on `ubuntu-latest` would compile DuckDB through cgo under QEMU; measured on 2026-10-12 that path is the expensive one (see Consequences). Instead `release.yml` builds each architecture on a **native** hosted runner (`ubuntu-latest` + `ubuntu-24.04-arm`), pushes by digest, and merges a manifest list. Native arm64 on the hosted runner was ~47s for a coldish image build; that is cheap enough to keep on every release. `multiarch-image.yml` exercises the same two native builds on PRs that touch the image path, without pushing.
+
 ### CI changes
 
-`ci.yml` now runs only on pushes to `main` and pull requests. It no longer publishes Docker images — that responsibility belongs entirely to `release.yml`. The smoke test continues to validate Docker builds on every CI run.
+`ci.yml` now runs only on pushes to `main` and pull requests. It no longer publishes Docker images — that responsibility belongs entirely to `release.yml`. The smoke test continues to validate an amd64 Docker build on every CI run; `multiarch-image.yml` covers the arm64 half of the release path.
 
 ## Options considered
 
@@ -50,8 +54,10 @@ Releases are created **only** by manually triggering `.github/workflows/release.
 
 - No Docker image is published until someone presses the button — deliberate gate.
 - GHCR always has a `latest` tag pointing to the newest release, and versioned tags for pinning.
+- `latest` (and each version tag) is a multi-arch manifest: `docker pull` on arm64 no longer requires a local `docker compose build` before `cotel --db-import`.
 - CHANGELOG must be kept up-to-date by the developer; the release workflow will produce an empty section if it isn't.
 - Branch protection on `main` would block the release workflow's push-back commit. If protection is added later, the workflow token must be granted bypass or a deploy key used instead.
+- QEMU-emulated arm64 on `ubuntu-latest` was measured and rejected for the release path: native `ubuntu-24.04-arm` finished the same Dockerfile in **47s**, while the QEMU build step was still running at **~11 minutes** when cancelled. Prefer native runners; do not add `platforms: linux/amd64,linux/arm64` on a single amd64 runner.
 
 ## Decision lenses applied