feat: publish deploy base images to DockerHub (#4581)

Publishes the base images deployed task containers will build on:
`triggerdotdev/node:{21,22,24,26}-bookworm` and
`triggerdotdev/bun:1.3-node20-bookworm`, each with a `-build` toolchain
variant, multi-arch, built from a pinned Debian snapshot so every
published layer is reproducible from recorded inputs and carries a
GitHub provenance attestation.

Publishing runs on manual dispatch and on merges touching
`base-images/`. Pull requests build without pushing.

Author: myftija

.github/workflows/base-images.yml

@@ -0,0 +1,263 @@
+name: 🐳 Deploy base images
+
+# Publishes the deploy base images (see base-images/README.md) to Docker Hub.
+# Tags are mutable and rebuilt in place; the CLI pins digests, so consumers
+# only move when a release bumps its pins.
+
+on:
+  workflow_dispatch:
+    inputs:
+      debian_snapshot:
+        description: "Debian snapshot timestamp (YYYYMMDDTHHMMSSZ). Defaults to yesterday 00:00 UTC."
+        required: false
+        type: string
+  push:
+    branches: [main]
+    paths:
+      - "base-images/**"
+      - ".github/workflows/base-images.yml"
+  pull_request:
+    paths:
+      - "base-images/**"
+      - ".github/workflows/base-images.yml"
+
+concurrency:
+  group: base-images-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions: {}
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    outputs:
+      images: ${{ steps.config.outputs.images }}
+      packages: ${{ steps.config.outputs.packages }}
+      build_packages: ${{ steps.config.outputs.build_packages }}
+      suite: ${{ steps.config.outputs.suite }}
+      snapshot: ${{ steps.config.outputs.snapshot }}
+      source_date_epoch: ${{ steps.config.outputs.source_date_epoch }}
+      push: ${{ steps.config.outputs.push }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Read image matrix and resolve snapshot
+        id: config
+        env:
+          SNAPSHOT_INPUT: ${{ inputs.debian_snapshot }}
+          EVENT_NAME: ${{ github.event_name }}
+          REF: ${{ github.ref }}
+        run: |
+          PACKAGES="$(jq -er '.packages' base-images/images.json)"
+          BUILD_PACKAGES="$(jq -er '.buildPackages' base-images/images.json)"
+          SUITE="$(jq -er '.suite' base-images/images.json)"
+
+          # Values land in build args and shell lines; keep them boring.
+          # NUL-delimited whole-record match so multi-line values can't sneak through
+          printf '%s\0' "$PACKAGES" | grep -zqxE '[a-z0-9][a-z0-9 .+:=~-]*' || { echo "invalid packages value"; exit 1; }
+          printf '%s\0' "$BUILD_PACKAGES" | grep -zqxE '[a-z0-9][a-z0-9 .+:=~-]*' || { echo "invalid buildPackages value"; exit 1; }
+          printf '%s\0' "$SUITE" | grep -zqxE '[a-z]+' || { echo "invalid suite value"; exit 1; }
+          jq -e '.images | length > 0 and all((.repo | test("^[a-z0-9-]+$")) and (.tag | test("^[a-z0-9.-]+$")) and (.base | test("^[a-zA-Z0-9./:@-]+$")))' base-images/images.json > /dev/null \
+            || { echo "invalid images entries"; exit 1; }
+
+          SNAPSHOT="$SNAPSHOT_INPUT"
+          if [ -z "$SNAPSHOT" ]; then
+            SNAPSHOT="$(date -u -d yesterday +%Y%m%dT000000Z)"
+          fi
+          printf '%s\0' "$SNAPSHOT" | grep -zqxE '[0-9]{8}T[0-9]{6}Z' || { echo "invalid debian_snapshot: $SNAPSHOT"; exit 1; }
+
+          # Snapshot-derived timestamps: reproducible, with a real created date
+          EPOCH="$(date -u -d "${SNAPSHOT:0:4}-${SNAPSHOT:4:2}-${SNAPSHOT:6:2} ${SNAPSHOT:9:2}:${SNAPSHOT:11:2}:${SNAPSHOT:13:2}Z" +%s)"
+          # Future snapshots resolve to "latest" and break mtime normalization
+          [ "$EPOCH" -le "$(date -u +%s)" ] || { echo "debian_snapshot is in the future: $SNAPSHOT"; exit 1; }
+
+          # Pull requests and branch dispatches build without pushing
+          if [ "$EVENT_NAME" = "pull_request" ] || [ "$REF" != "refs/heads/main" ]; then
+            PUSH=false
+          else
+            PUSH=true
+          fi
+
+          {
+            echo "images=$(jq -c '.images' base-images/images.json)"
+            echo "packages=$PACKAGES"
+            echo "build_packages=$BUILD_PACKAGES"
+            echo "suite=$SUITE"
+            echo "snapshot=$SNAPSHOT"
+            echo "source_date_epoch=$EPOCH"
+            echo "push=$PUSH"
+          } >> "$GITHUB_OUTPUT"
+
+  publish:
+    needs: setup
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJSON(needs.setup.outputs.images) }}
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKER_BUILD_SUMMARY: "false"
+      DOCKER_BUILD_RECORD_UPLOAD: "false"
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      # Before any pull so rate limits are authenticated; fork PRs skip (no secrets)
+      - name: 🐳 Login to Docker Hub
+        if: env.DOCKERHUB_USERNAME != ''
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🐳 Set up QEMU
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+        with:
+          image: docker.io/tonistiigi/binfmt:latest@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0
+
+      - name: 🐳 Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      # Build both targets before pushing either so the tag pair can't skew
+      - name: 🐳 Build both targets (no push)
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: base-images
+          file: base-images/Dockerfile
+          target: build
+          platforms: linux/amd64,linux/arm64
+          provenance: false
+          outputs: type=image,push=false,rewrite-timestamp=true
+          tags: triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}-build
+          build-args: |
+            BASE_IMAGE=${{ matrix.image.base }}
+            DEBIAN_SNAPSHOT=${{ needs.setup.outputs.snapshot }}
+            DEBIAN_SUITE=${{ needs.setup.outputs.suite }}
+            PACKAGES=${{ needs.setup.outputs.packages }}
+            BUILD_PACKAGES=${{ needs.setup.outputs.build_packages }}
+            SOURCE_DATE_EPOCH=${{ needs.setup.outputs.source_date_epoch }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            dev.trigger.debian-snapshot=${{ needs.setup.outputs.snapshot }}
+
+      - name: 🐳 Push runtime image
+        id: build_runtime
+        if: needs.setup.outputs.push == 'true'
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: base-images
+          file: base-images/Dockerfile
+          target: runtime
+          platforms: linux/amd64,linux/arm64
+          provenance: false
+          outputs: type=image,push=true,rewrite-timestamp=true
+          tags: triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}
+          build-args: |
+            BASE_IMAGE=${{ matrix.image.base }}
+            DEBIAN_SNAPSHOT=${{ needs.setup.outputs.snapshot }}
+            DEBIAN_SUITE=${{ needs.setup.outputs.suite }}
+            PACKAGES=${{ needs.setup.outputs.packages }}
+            SOURCE_DATE_EPOCH=${{ needs.setup.outputs.source_date_epoch }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            dev.trigger.debian-snapshot=${{ needs.setup.outputs.snapshot }}
+
+      - name: 🐳 Push build-variant image
+        id: build_toolchain
+        if: needs.setup.outputs.push == 'true'
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: base-images
+          file: base-images/Dockerfile
+          target: build
+          platforms: linux/amd64,linux/arm64
+          provenance: false
+          outputs: type=image,push=true,rewrite-timestamp=true
+          tags: triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}-build
+          build-args: |
+            BASE_IMAGE=${{ matrix.image.base }}
+            DEBIAN_SNAPSHOT=${{ needs.setup.outputs.snapshot }}
+            DEBIAN_SUITE=${{ needs.setup.outputs.suite }}
+            PACKAGES=${{ needs.setup.outputs.packages }}
+            BUILD_PACKAGES=${{ needs.setup.outputs.build_packages }}
+            SOURCE_DATE_EPOCH=${{ needs.setup.outputs.source_date_epoch }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            dev.trigger.debian-snapshot=${{ needs.setup.outputs.snapshot }}
+
+      # An auto-created private repo would publish green while customer pulls fail
+      - name: 🔎 Verify anonymous pullability
+        if: needs.setup.outputs.push == 'true'
+        env:
+          IMAGE_REPO: ${{ matrix.image.repo }}
+          RUNTIME_DIGEST: ${{ steps.build_runtime.outputs.digest }}
+          BUILD_DIGEST: ${{ steps.build_toolchain.outputs.digest }}
+        run: |
+          for digest in "$RUNTIME_DIGEST" "$BUILD_DIGEST"; do
+            TOKEN="$(curl -fsS --connect-timeout 10 --max-time 60 "https://auth.docker.io/token?service=registry.docker.io&scope=repository:triggerdotdev/$IMAGE_REPO:pull" | jq -r .token)"
+            curl -fsS --connect-timeout 10 --max-time 60 -o /dev/null               -H "Authorization: Bearer $TOKEN"               -H "Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"               "https://registry-1.docker.io/v2/triggerdotdev/$IMAGE_REPO/manifests/$digest"               || { echo "triggerdotdev/$IMAGE_REPO@$digest is not anonymously pullable; is the repo private?"; exit 1; }
+          done
+
+      # Builds are reproducible, so re-running a red publish re-pushes the
+      # same digests and re-attests them
+      - name: 🔏 Attest runtime image provenance
+        if: needs.setup.outputs.push == 'true'
+        uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
+        with:
+          subject-name: index.docker.io/triggerdotdev/${{ matrix.image.repo }}
+          subject-digest: ${{ steps.build_runtime.outputs.digest }}
+          push-to-registry: false
+
+      - name: 🔏 Attest build-variant image provenance
+        if: needs.setup.outputs.push == 'true'
+        uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
+        with:
+          subject-name: index.docker.io/triggerdotdev/${{ matrix.image.repo }}
+          subject-digest: ${{ steps.build_toolchain.outputs.digest }}
+          push-to-registry: false
+
+      - name: 📋 Record digests
+        if: needs.setup.outputs.push == 'true'
+        env:
+          IMAGE_REPO: ${{ matrix.image.repo }}
+          IMAGE_TAG: ${{ matrix.image.tag }}
+          RUNTIME_DIGEST: ${{ steps.build_runtime.outputs.digest }}
+          BUILD_DIGEST: ${{ steps.build_toolchain.outputs.digest }}
+          SNAPSHOT: ${{ needs.setup.outputs.snapshot }}
+        run: |
+          {
+            echo "### triggerdotdev/$IMAGE_REPO:$IMAGE_TAG"
+            echo '```'
+            echo "runtime: $RUNTIME_DIGEST"
+            echo "build:   $BUILD_DIGEST"
+            echo "debian snapshot: $SNAPSHOT"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  results:
+    needs: [publish]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions: {}
+    steps:
+      - name: Fail if any image build failed
+        env:
+          RESULT: ${{ needs.publish.result }}
+        run: |
+          [ "$RESULT" = "success" ] || { echo "one or more image builds failed: $RESULT"; exit 1; }

base-images/Dockerfile

@@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1
+# check=skip=InvalidDefaultArgInFrom
+
+# Base images for deployed task containers; see README.md. Packages install
+# from a pinned Debian snapshot, then apt is restored to the live archive.
+
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE} AS runtime
+
+ARG DEBIAN_SNAPSHOT
+ARG DEBIAN_SUITE=bookworm
+ARG PACKAGES
+
+# ARG, not ENV: needed during build only, must not leak into task containers
+ARG DEBIAN_FRONTEND=noninteractive
+
+# http, not https: the slim bases have no ca-certificates yet, and apt
+# integrity comes from GPG-signed Release files rather than TLS.
+# check-valid-until=no: pinned Release files outlive their Valid-Until window.
+RUN . /etc/os-release && [ "$VERSION_CODENAME" = "${DEBIAN_SUITE}" ] || { echo "Base image is Debian $VERSION_CODENAME but this build pins ${DEBIAN_SUITE} apt sources"; exit 1; } && \
+  [ -n "${DEBIAN_SNAPSHOT}" ] && [ -n "${PACKAGES}" ] || { echo "DEBIAN_SNAPSHOT and PACKAGES build args are required (see images.json)"; exit 1; } && \
+  mv /etc/apt/sources.list.d/debian.sources /tmp/debian.sources && \
+  { [ ! -f /etc/apt/sources.list ] || mv /etc/apt/sources.list /tmp/upstream-sources.list; } && \
+  printf '%s\n' \
+    "deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE} main" \
+    "deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-security main" \
+    "deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-updates main" \
+    > /etc/apt/sources.list.d/snapshot.list && \
+  printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\n' > /etc/apt/apt.conf.d/99-snapshot-retries && \
+  apt-get update && \
+  apt-get upgrade -y --with-new-pkgs && \
+  apt-get install -y --no-install-recommends ${PACKAGES} && \
+  apt-get clean && \
+  rm /etc/apt/sources.list.d/snapshot.list /etc/apt/apt.conf.d/99-snapshot-retries && \
+  mv /tmp/debian.sources /etc/apt/sources.list.d/debian.sources && \
+  { [ ! -f /tmp/upstream-sources.list ] || mv /tmp/upstream-sources.list /etc/apt/sources.list; } && \
+  rm -rf /var/lib/apt/lists/* /var/log/dpkg.log /var/log/apt /var/log/alternatives.log /var/cache/ldconfig/aux-cache /var/cache/debconf/*-old
+
+FROM runtime AS build
+
+ARG DEBIAN_SNAPSHOT
+ARG DEBIAN_SUITE=bookworm
+ARG BUILD_PACKAGES
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN [ -n "${DEBIAN_SNAPSHOT}" ] && [ -n "${BUILD_PACKAGES}" ] || { echo "DEBIAN_SNAPSHOT and BUILD_PACKAGES build args are required (see images.json)"; exit 1; } && \
+  mv /etc/apt/sources.list.d/debian.sources /tmp/debian.sources && \
+  { [ ! -f /etc/apt/sources.list ] || mv /etc/apt/sources.list /tmp/upstream-sources.list; } && \
+  printf '%s\n' \
+    "deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE} main" \
+    "deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-security main" \
+    "deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-updates main" \
+    > /etc/apt/sources.list.d/snapshot.list && \
+  printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\n' > /etc/apt/apt.conf.d/99-snapshot-retries && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends ${BUILD_PACKAGES} && \
+  apt-get clean && \
+  rm /etc/apt/sources.list.d/snapshot.list /etc/apt/apt.conf.d/99-snapshot-retries && \
+  mv /tmp/debian.sources /etc/apt/sources.list.d/debian.sources && \
+  { [ ! -f /tmp/upstream-sources.list ] || mv /tmp/upstream-sources.list /etc/apt/sources.list; } && \
+  rm -rf /var/lib/apt/lists/* /var/log/dpkg.log /var/log/apt /var/log/alternatives.log /var/cache/ldconfig/aux-cache /var/cache/debconf/*-old

base-images/README.md

@@ -0,0 +1,72 @@
+# Deploy base images
+
+Base images for deployed task containers, published to Docker Hub as
+`triggerdotdev/node:<major>-bookworm` and `triggerdotdev/bun:<line>-node<major>-bookworm`,
+each with a `-build` variant that adds the native-module toolchain
+(python3, make, g++).
+
+Each image is its upstream slim base (pinned by digest in `images.json`) with
+all preinstalled Debian packages upgraded to the pinned snapshot state, plus
+the system packages deployed tasks rely on: busybox, ca-certificates,
+dumb-init, git, openssl. apt stays configured for the live Debian archive, so
+images derived from these behave like their upstream bases.
+
+## Tags and pinning
+
+Tags are mutable and rebuilt in place on demand; each rebuild picks up Debian
+security updates published up to its snapshot date. The
+runtime itself (the node or bun binaries from the upstream base) only moves
+when the base digests in `images.json` are bumped. When bumping a base
+digest, keep the snapshot at least as new as the upstream image's own archive
+state, or the upgrade step silently becomes a no-op. Consumers pin digests: the CLI's generated
+Containerfile references these images as `triggerdotdev/node:22-bookworm@sha256:...`,
+and digests only move when a CLI release updates its pins.
+
+## Reproducibility and provenance
+
+Packages install from a [Debian snapshot archive](https://snapshot.debian.org)
+timestamp recorded in the `dev.trigger.debian-snapshot` image label, and the
+workflow exports layers with timestamps normalized to the snapshot date
+(`SOURCE_DATE_EPOCH` plus `rewrite-timestamp`), so a published image's layers
+are a pure function of
+(upstream base digest, snapshot timestamp, package list). To verify, rebuild
+with the recorded inputs and compare layer digests (the manifest and config
+digests differ because they carry build metadata labels like the source
+revision):
+
+```bash
+# needs a docker-container builder (docker buildx create --use)
+SNAPSHOT=$(docker buildx imagetools inspect triggerdotdev/node:22-bookworm \
+  --format '{{index (index .Image "linux/amd64").Config.Labels "dev.trigger.debian-snapshot"}}')
+# GNU date; the epoch must match the one the workflow derived from the snapshot
+EPOCH=$(date -u -d "${SNAPSHOT:0:4}-${SNAPSHOT:4:2}-${SNAPSHOT:6:2} ${SNAPSHOT:9:2}:${SNAPSHOT:11:2}:${SNAPSHOT:13:2}Z" +%s)
+docker buildx build base-images --target runtime \
+  --build-arg BASE_IMAGE="<base from images.json>" \
+  --build-arg PACKAGES="<packages from images.json>" \
+  --build-arg DEBIAN_SNAPSHOT="$SNAPSHOT" \
+  --build-arg SOURCE_DATE_EPOCH="$EPOCH" \
+  --platform linux/amd64,linux/arm64 \
+  --provenance false \
+  --output type=oci,dest=rebuilt.tar,rewrite-timestamp=true
+# then compare .layers[].digest of the rebuilt per-platform manifests against
+# the published ones (imagetools inspect --raw returns the index; fetch each
+# platform manifest it references to see its layers). For the -build variant,
+# use --target build and additionally pass --build-arg BUILD_PACKAGES. Layer
+# digests are stable for a given BuildKit version and compression settings.
+```
+
+Every published digest carries a GitHub build provenance attestation (a
+publish whose attestation fails goes red and is re-run):
+
+```bash
+gh attestation verify oci://index.docker.io/triggerdotdev/node:22-bookworm \
+  --repo triggerdotdev/trigger.dev \
+  --signer-workflow triggerdotdev/trigger.dev/.github/workflows/base-images.yml
+```
+
+## Publishing
+
+`.github/workflows/base-images.yml` publishes on manual dispatch and on
+changes to this directory. Pull requests build the images
+without pushing. After a publish, the digests in the job summary are used to
+update the `BASE_IMAGE` pins in `packages/cli-v3/src/deploy/buildImage.ts`.