diff --git a/.github/actions/cache-mount/action.yml b/.github/actions/cache-mount/action.yml new file mode 100644 index 00000000000..cf4d0276507 --- /dev/null +++ b/.github/actions/cache-mount/action.yml @@ -0,0 +1,40 @@ +name: Cache Mount +description: Mount a build cache directory using Blacksmith sticky disks, or the GitHub Actions cache when running on GitHub-hosted runners. + +inputs: + provider: + description: Empty or 'blacksmith' selects sticky disks. Any other value selects actions/cache. Must stay byte-identical to the runs-on predicate in the calling workflow. + required: false + default: '' + key: + description: Cache key. Shared by both backends, so callers keep one key regardless of provider. + required: true + path: + description: Filesystem path the cache is mounted at. + required: true + +# The two conditions are exact complements of the runs-on expression the caller +# uses. They must be inverted together: a job placed on ubuntu-latest that then +# took the sticky-disk branch would hard-fail, since stickydisk hot-mounts an +# ext4 volume from Blacksmith's storage agents and cannot run anywhere else. +runs: + using: composite + steps: + - name: Mount sticky disk + if: inputs.provider == '' || inputs.provider == 'blacksmith' + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ inputs.key }} + path: ${{ inputs.path }} + + # Sticky disks always re-commit the mounted path, so the cache rolls forward + # every run. actions/cache skips its save step on an exact key hit, which + # would freeze the cache at whatever the first run wrote — the run-scoped + # suffix plus a prefix restore-key reproduces the rolling behaviour. + - name: Restore and save cache + if: inputs.provider != '' && inputs.provider != 'blacksmith' + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + key: ${{ inputs.key }}-${{ github.run_id }} + restore-keys: ${{ inputs.key }}- + path: ${{ inputs.path }} diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml new file mode 100644 index 00000000000..600f4a45582 --- /dev/null +++ b/.github/actions/docker-build/action.yml @@ -0,0 +1,64 @@ +name: Docker Build and Push +description: Set up a buildx builder and build/push an image, using Blacksmith's builder or the upstream Docker actions on GitHub-hosted runners. + +inputs: + provider: + description: Empty or 'blacksmith' selects Blacksmith's builder. Any other value selects the upstream docker/* actions. Must stay byte-identical to the runs-on predicate in the calling workflow. + required: false + default: '' + context: + description: Build context. + required: false + default: . + file: + description: Path to the Dockerfile. + required: true + platforms: + description: Target platforms, e.g. linux/amd64. + required: true + tags: + description: Comma-separated list of tags to push. + required: true + +# Registry logins must already have run in the calling job — both backends read +# the same ~/.docker/config.json. provenance/sbom stay off everywhere: the +# attestation manifests they add break `imagetools create` retagging in the +# promote-images and create-ghcr-manifests jobs. +runs: + using: composite + steps: + - name: Set up Blacksmith builder + if: inputs.provider == '' || inputs.provider == 'blacksmith' + uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 + + - name: Build and push (Blacksmith) + if: inputs.provider == '' || inputs.provider == 'blacksmith' + uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + push: true + tags: ${{ inputs.tags }} + provenance: false + sbom: false + + - name: Set up Docker Buildx + if: inputs.provider != '' && inputs.provider != 'blacksmith' + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + + # No layer cache here. Blacksmith caches layers on the builder itself; + # the GitHub equivalent (cache-to: type=gha) shares the same 10 GB repo + # cache quota as the bun/node_modules/turbo mounts, and four images of + # layers would evict them. Fallback builds are cold by design. + - name: Build and push (GitHub) + if: inputs.provider != '' && inputs.provider != 'blacksmith' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + push: true + tags: ${{ inputs.tags }} + provenance: false + sbom: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca2901c7871..9da49fcfce6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,33 @@ name: CI +# Runner provider toggle. Every job here and in the reusable workflows reads the +# CI_PROVIDER repository variable: +# +# gh variable set CI_PROVIDER --body github # fall back to GitHub-hosted +# gh variable delete CI_PROVIDER # back to Blacksmith (default) +# +# It is a repo variable rather than a committed value on purpose: during a +# Blacksmith outage there is no working CI to merge a switchover commit through, +# and the variable takes effect on the next run with nothing to land. +# +# Only unset or the exact value 'blacksmith' selects Blacksmith; everything else +# selects GitHub-hosted. The unrecognized case resolves toward GitHub on purpose, +# because that is the provider that is always available: a typo'd or whitespace- +# padded break-glass value then yields slow-but-working CI instead of jobs queued +# forever against the provider you were trying to escape. `==` is case-insensitive +# in Actions expressions, so 'GitHub' and 'Blacksmith' both work. Every runs-on +# expression and both composite actions use this same predicate — they must be +# changed together, since a job on ubuntu-latest that took a Blacksmith-only +# branch would hard-fail. +# +# GitHub mode is a break-glass fallback, not a peer. Public-repo ubuntu-latest is +# 4 vCPU with no free 8-core tier, the 10 GB repo cache quota will thrash across +# the bun/node_modules/turbo mounts, and image builds start from cold layers — so +# expect slower runs, not different results. Blacksmith-only actions are reached +# exclusively through .github/actions/cache-mount and .github/actions/docker-build, +# which branch internally; a bare useblacksmith/* step added anywhere else will +# hard-fail in GitHub mode rather than silently degrade. + on: push: branches: [main, staging, dev] @@ -28,7 +56,7 @@ jobs: # Detect if this is a version release commit (e.g., "v0.5.24: ...") detect-version: name: Detect Version - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/dev') outputs: @@ -81,7 +109,7 @@ jobs: name: Build Dev ECR needs: [detect-version, migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 30 permissions: contents: read @@ -118,9 +146,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 - - name: Resolve ECR repo name id: ecr-repo run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT @@ -128,15 +153,12 @@ jobs: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} - name: Build and push - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: ./.github/actions/docker-build with: - context: . + provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - push: true tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev - provenance: false - sbom: false # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch. # Gated after migrate-dev for the same reason as build-dev — the new task @@ -145,7 +167,7 @@ jobs: name: Deploy Trigger.dev (Dev) needs: [migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 steps: - name: Checkout code @@ -192,7 +214,7 @@ jobs: if: >- github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 30 permissions: contents: read @@ -242,9 +264,6 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 - - name: Resolve ECR repo name id: ecr-repo run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT @@ -270,15 +289,12 @@ jobs: echo "tags=${TAGS}" >> $GITHUB_OUTPUT - name: Build and push images - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: ./.github/actions/docker-build with: - context: . + provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - push: true tags: ${{ steps.meta.outputs.tags }} - provenance: false - sbom: false # Promote the sha-tagged ECR images to the deploy tags once tests and # migrations pass. Pushing the ECR latest/staging tag is what triggers @@ -292,7 +308,7 @@ jobs: if: >- github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') - runs-on: blacksmith-2vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -362,7 +378,7 @@ jobs: # never moves a documented tag. build-ghcr-arm64: name: Build ARM64 (GHCR Only) - runs-on: blacksmith-8vcpu-ubuntu-2404-arm + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404-arm' || 'ubuntu-24.04-arm' }} timeout-minutes: 30 if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: @@ -392,26 +408,20 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 - - name: Build and push ARM64 to GHCR - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: ./.github/actions/docker-build with: - context: . + provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/arm64 - push: true tags: ${{ matrix.image }}:${{ github.sha }}-arm64 - provenance: false - sbom: false # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags) # and the multi-arch manifests from the immutable sha tags — only on main, # after the deploy gate (promote-images) and the ARM64 build both pass. create-ghcr-manifests: name: Create GHCR Manifests - runs-on: blacksmith-2vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 10 needs: [promote-images, build-ghcr-arm64, detect-version] if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -477,7 +487,7 @@ jobs: # Check if docs changed check-docs-changes: name: Check Docs Changes - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 if: github.event_name == 'push' && github.ref == 'refs/heads/main' outputs: @@ -506,7 +516,7 @@ jobs: # Create GitHub Release (only for version commits on main, after all builds complete) create-release: name: Create GitHub Release - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 10 needs: [create-ghcr-manifests, detect-version] if: needs.detect-version.outputs.is_release == 'true' diff --git a/.github/workflows/docs-embeddings.yml b/.github/workflows/docs-embeddings.yml index 76cf58c584e..b1eb9a28f49 100644 --- a/.github/workflows/docs-embeddings.yml +++ b/.github/workflows/docs-embeddings.yml @@ -10,7 +10,7 @@ permissions: jobs: process-docs-embeddings: name: Process Documentation Embeddings - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 30 if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 593bf854a4e..ab177ed0c5d 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -24,7 +24,7 @@ permissions: jobs: migrate: name: Apply Database Migrations - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 45 steps: diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index da24a3959be..54f8b0b7038 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -11,7 +11,7 @@ permissions: jobs: publish-npm: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 steps: - name: Checkout repository diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml index 1890c10b9f5..98e598b11d8 100644 --- a/.github/workflows/publish-python-sdk.yml +++ b/.github/workflows/publish-python-sdk.yml @@ -11,7 +11,7 @@ permissions: jobs: publish-pypi: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 steps: - name: Checkout repository diff --git a/.github/workflows/publish-ts-sdk.yml b/.github/workflows/publish-ts-sdk.yml index f64f3867a74..e6bb8891b5f 100644 --- a/.github/workflows/publish-ts-sdk.yml +++ b/.github/workflows/publish-ts-sdk.yml @@ -11,7 +11,7 @@ permissions: jobs: publish-npm: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 steps: - name: Checkout repository diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index f20cae5f7d6..6e49b97c220 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -10,7 +10,7 @@ permissions: jobs: test-build: name: Lint and Test - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 steps: @@ -27,25 +27,28 @@ jobs: with: node-version: 22 - # Sticky-disk keys are scoped by event name, and fork PRs get their own - # namespace on top: untrusted fork runs must never share a disk with - # push runs (whose disks feed production image builds) or with trusted - # internal-PR runs. - - name: Mount Bun cache (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + # Cache keys are scoped by event name, and fork PRs get their own + # namespace on top: untrusted fork runs must never share a cache with + # push runs (whose caches feed production image builds) or with trusted + # internal-PR runs. The scoping holds on both backends. + - name: Mount Bun cache + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ~/.bun/install/cache - - name: Mount node_modules (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount node_modules + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./node_modules - - name: Mount Turbo cache (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount Turbo cache + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-turbo-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./.turbo @@ -185,10 +188,11 @@ jobs: # with test-build (identical content from the same lockfile — LWW loss is # harmless), but the Turbo cache gets its own key: with a shared key, only # the last committer's new entries survive each run, so the test and build - # Turbo entries would evict each other nondeterministically. + # Turbo entries would evict each other nondeterministically. The same split + # is correct on actions/cache, which is likewise last-writer-wins per key. build: name: Build App - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 steps: @@ -205,21 +209,24 @@ jobs: with: node-version: 22 - - name: Mount Bun cache (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount Bun cache + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ~/.bun/install/cache - - name: Mount node_modules (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount node_modules + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./node_modules - - name: Mount Turbo cache (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount Turbo cache + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-turbo-cache-build-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./.turbo