docker.yml

name: Docker Container

# TC-312: the release -> auto-deploy chain used to fire on the GitHub Release
# `published` event; it no longer does. release-plz.toml sets
# `git_release_enable = false`, so release-plz only pushes the `vX.Y.Z` tag —
# the GitHub Release itself is created by cargo-dist (release.yml) with the
# default `GITHUB_TOKEN`. GitHub deliberately does NOT emit workflow-triggering
# events (including `release`) for actions taken with `GITHUB_TOKEN`, to prevent
# recursive runs, so `on: release` here never fires from a cargo-dist release
# and prod could only be deployed by a manual `workflow_dispatch`.
#
# Fix: trigger on the tag push itself. release-plz pushes tags with a PAT
# (RELEASE_PLZ_TOKEN) and a manual `git push --tags` also emits a normal `push`
# event, so tag pushes reliably reach this workflow. `on: release` is kept for
# backward compatibility should a PAT-published Release ever reappear.
on:
  push:
    branches: [main]
    tags: ['v*']
  pull_request:
    branches: [main]
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      rollback_audit_only:
        description: 'Read-only aggregate meeting rollback inventory; skips image builds and deployment'
        required: false
        type: boolean
        default: false
      include_duckdb:
        description: 'Build images with DuckDB support enabled'
        required: false
        type: boolean
        default: false
      include_tc_bench_v1:
        description: 'Build images with the tc-bench-v1 endpoints enabled'
        required: false
        type: boolean
        default: false
      deploy_phala:
        description: 'Deploy the dstack image to Phala after build'
        required: false
        type: boolean
        default: false
      authorize_quota_values:
        description: 'Fresh authorization of the reviewed quota values/authority for this deployment; required on every deploy_phala path.'
        required: false
        type: boolean
        default: false
      rollback_pre_n0:
        description: 'Deploy only the captured pre-N0 1.20.6 compose; requires an explicit Sam-approved rollback privacy mode.'
        required: false
        type: boolean
        default: false
      rollback_privacy_mode:
        description: 'Explicit rollback-only policy; stop-future-local-logs does NOT delete existing files. No production default.'
        required: false
        type: choice
        options: [unselected, keep-json-file, stop-future-local-logs]
        default: unselected
      allow_non_descendant:
        description: 'Skip the production ancestry guard for a deliberate non-descendant deployment'
        required: false
        type: boolean
        default: false
      sql_identity_fence:
        description: 'Keep SQL/DuckDB fenced in the Phala deploy (true until offline verification passes)'
        required: true
        type: boolean
        default: true
      image_version:
        description: 'Image version without leading v. Builds must match the selected crate; digest promotions must match the recorded image version (rollback_image_version when supplied).'
        required: false
        type: string
        default: ''
      deploy_image_digest:
        description: 'Promote an existing sha256 digest without rebuilding, dispatched on main only. Requires truthful image_version, deploy_phala and default dstack features.'
        required: false
        type: string
        default: ''
      rollback_image_version:
        description: 'Recorded immutable digest version label, for recovery, rollback or forward digest promotion; revision must be in main history'
        required: false
        type: string
        default: ''
      rollback_image_revision:
        description: 'Recorded digest 40-character revision label (must be an ancestor of main)'
        required: false
        type: string
        default: ''

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: tinycloudlabs/tinycloud-node

jobs:
  rollback-audit:
    if: github.event_name == 'workflow_dispatch' && inputs.rollback_audit_only
    runs-on: ubuntu-latest
    # TC-971: PROD_TINYCLOUD_DATABASE_URL lives in the production environment.
    environment: production
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4
      - name: Audit durable meeting publication footprint without exporting data
        env:
          RECOVERY_DATABASE_URL: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }}
          # libpq needs the public CA bundle explicitly for verify-full;
          # production's Rust TLS client has compiled-in public roots.
          PGSSLROOTCERT: /etc/ssl/certs/ca-certificates.crt
        run: python3 scripts/audit-meeting-rollback.py
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: meeting-rollback-audit
          path: meeting-rollback-audit.json
          if-no-files-found: ignore
  # TC-342: every image tag this workflow can produce must agree with the crate
  # version at the checked-out ref. `image_version` used to be unconstrained
  # free text, which is how an image got labelled 1.10.0 from a tree whose
  # Cargo.toml said 1.9.0 — prod then ran a binary that no tag in the repo
  # identified, and the team debugged a feature that was not in the deployed
  # image. Fail the whole workflow loudly instead of shipping a mislabelled tag.
  version-guard:
    name: Version guard
    runs-on: ubuntu-latest
    outputs:
      image_version: ${{ steps.check.outputs.image_version }}
    steps:
      - uses: actions/checkout@v4
      - name: Check published Docker tags
        run: python3 scripts/ci/test-docker-tags.py

      - name: Test quota authority with synthetic HTTP
        run: python3 scripts/ci/test-quota-authority.py
      - name: Test recovery deployment boundaries and documented commands
        run: python3 scripts/ci/test-deploy-recovery.py

      - name: Validate image version against Cargo.toml
        id: check
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUT_IMAGE_VERSION: ${{ inputs.image_version }}
          INPUT_DEPLOY_DIGEST: ${{ inputs.deploy_image_digest }}
          INPUT_DEPLOY_PHALA: ${{ inputs.deploy_phala }}
          AUTHORIZE_QUOTA_VALUES: ${{ inputs.authorize_quota_values }}
          ROLLBACK_PRE_N0: ${{ inputs.rollback_pre_n0 }}
          ROLLBACK_PRIVACY_MODE: ${{ inputs.rollback_privacy_mode }}
          APPROVED_ROLLBACK_PRIVACY_MODE: ${{ vars.TC959_ROLLBACK_PRIVACY_MODE }}
          ROLLBACK_VERSION: ${{ inputs.rollback_image_version }}
          ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }}
          INCLUDE_DUCKDB: ${{ inputs.include_duckdb }}
          INCLUDE_TC_BENCH_V1: ${{ inputs.include_tc_bench_v1 }}
          RELEASE_TAG: ${{ github.event.release.tag_name }}
        run: |
          set -euo pipefail
          # #310 and this recovery change must land as one lineage, before
          # #309's later logging window. Never merge #310 alone.
          if [ "${INPUT_DEPLOY_PHALA}" = "true" ]; then
            if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${AUTHORIZE_QUOTA_VALUES:-false}" != "true" ]; then
              echo "::error::Every deployment requires explicit authorize_quota_values=true before any mutation"
              exit 1
            fi
          fi
          python3 scripts/ci/reconcile-quota.py --validate-only

          # `|| true` so a missing match reaches the explicit check below
          # instead of dying under `set -e` with no explanation.
          CARGO_VERSION="$(grep -m1 -E '^version = "' tinycloud-node-server/Cargo.toml | cut -d '"' -f 2 || true)"
          if [ -z "${CARGO_VERSION}" ]; then
            echo "::error::could not read the tinycloud-node version from tinycloud-node-server/Cargo.toml"
            exit 1
          fi
          if [ "${ROLLBACK_PRE_N0:-false}" = "true" ]; then
            if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${GITHUB_REF}" != "refs/heads/main" ] \
              || [ "${INPUT_DEPLOY_PHALA}" != "true" ] \
              || [ "${INPUT_DEPLOY_DIGEST}" != "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" ] \
              || [ "${INPUT_IMAGE_VERSION}" != "1.20.6" ] || [ "${ROLLBACK_VERSION}" != "1.20.6" ] \
              || [ "${ROLLBACK_REVISION}" != "e18578ff9162986ddd813259c60f73866e3e60d1" ]; then
              echo "::error::Captured pre-N0 rollback requires main-only dispatch and the exact recorded 1.20.6 digest labels"
              exit 1
            fi
            case "${ROLLBACK_PRIVACY_MODE:-unselected}" in
              keep-json-file|stop-future-local-logs) ;;
              *) echo "::error::Choose an explicit Sam-approved rollback privacy mode"; exit 1 ;;
            esac
            if [ "${APPROVED_ROLLBACK_PRIVACY_MODE:-}" != "${ROLLBACK_PRIVACY_MODE}" ]; then
              echo "::error::Rollback privacy mode does not match the recorded Sam card decision"
              exit 1
            fi
          elif [ "${ROLLBACK_PRIVACY_MODE:-unselected}" != "unselected" ]; then
            echo "::error::Rollback privacy selection is mutually exclusive from current-compose promotion"
            exit 1
          fi

          check_matches() {
            label="$1"
            value="$2"
            if [ "${value#v}" != "${CARGO_VERSION}" ]; then
              echo "::error::${label} is '${value}' but tinycloud-node-server/Cargo.toml at this ref says '${CARGO_VERSION}'. Image tags must match the crate version — bump Cargo.toml via the release-plz release PR first (see docs/releasing.md)."
              exit 1
            fi
            echo "${label} '${value}' matches the crate version."
          }

          if [ "${EVENT_NAME}" = "release" ]; then
            check_matches "release tag" "${RELEASE_TAG}"
          elif [ "${GITHUB_REF_TYPE}" = "tag" ]; then
            check_matches "git tag" "${GITHUB_REF_NAME}"
          fi

          # TC-940: every production revision is in main's history, so a
          # rollback dispatched on main passes the ancestry guard without
          # allow_non_descendant. The rollback revision itself must be an
          # ancestor of main (checked by scripts/ci/check-deploy-ref.sh).
          if [ -n "${ROLLBACK_VERSION}" ] || [ -n "${ROLLBACK_REVISION}" ]; then
            if [ -z "${INPUT_DEPLOY_DIGEST}" ] \
              || ! [[ "${ROLLBACK_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
              || ! [[ "${ROLLBACK_REVISION}" =~ ^[0-9a-f]{40}$ ]]; then
              echo "::error::Rollback label override requires a digest and valid version/revision labels"
              exit 1
            fi
            echo "::notice::Using recorded image labels: ${ROLLBACK_VERSION} revision ${ROLLBACK_REVISION}"
          fi
          if [ -n "${INPUT_IMAGE_VERSION}" ]; then
            if [ -n "${INPUT_DEPLOY_DIGEST}" ] && [ -n "${ROLLBACK_VERSION}" ]; then
              if [ "${INPUT_IMAGE_VERSION}" != "${ROLLBACK_VERSION}" ]; then
                echo "::error::Digest promotion image_version must equal the recorded image version"
                exit 1
              fi
            else
              check_matches "image_version input" "${INPUT_IMAGE_VERSION}"
            fi
          fi
          if [ -n "${INPUT_DEPLOY_DIGEST}" ]; then
            if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${GITHUB_REF}" != "refs/heads/main" ] \
              || [ "${INPUT_DEPLOY_PHALA}" != "true" ] || [ -z "${INPUT_IMAGE_VERSION}" ] \
              || [ "${INCLUDE_DUCKDB}" = "true" ] || [ "${INCLUDE_TC_BENCH_V1}" = "true" ] \
              || ! [[ "${INPUT_DEPLOY_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
              echo "::error::Digest promotion requires workflow_dispatch on main, deploy_phala=true, truthful image_version, default dstack features and a sha256 digest."
              exit 1
            fi
          fi
          # Only the version of the selected binary enters deployment
          # outputs/logs; the workflow checkout is not the promoted image.
          SELECTED_VERSION="${ROLLBACK_VERSION:-${CARGO_VERSION}}"
          echo "image_version=${SELECTED_VERSION}" >> "$GITHUB_OUTPUT"
          echo "Selected image version: ${SELECTED_VERSION}"

      # TC-940: refuse a production deploy from anything but main or a vX.Y.Z
      # tag on main's history before spending time on image builds. The
      # deploy-phala job repeats this check next to the secrets.
      - name: Guard main-only deploy source
        if: github.event_name == 'workflow_dispatch' && inputs.deploy_phala
        env:
          ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }}
        run: |
          set -euo pipefail
          git fetch --quiet --no-tags --unshallow origin '+refs/heads/main:refs/remotes/origin/main'
          bash scripts/ci/check-deploy-ref.sh

  build:
    runs-on: ubuntu-latest
    needs: [version-guard]
    if: vars.TC780_CUTOVER_READY == 'true' && inputs.deploy_image_digest == '' && !inputs.rollback_audit_only
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to GHCR
        if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Resolve optional build features
        id: build_features
        run: |
          FEATURES=""
          IMAGE_SUFFIX=""
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then
            FEATURES="duckdb"
            IMAGE_SUFFIX="-duckdb"
          fi
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_tc_bench_v1 }}" = "true" ]; then
            FEATURES="${FEATURES:+$FEATURES }tc-bench-v1"
            IMAGE_SUFFIX="${IMAGE_SUFFIX}-tc-bench-v1"
          fi
          {
            echo "cargo_features=${FEATURES}"
            echo "image_suffix=${IMAGE_SUFFIX}"
            echo "manual_version=${{ inputs.image_version }}"
          } >> "$GITHUB_OUTPUT"

      - name: Extract metadata (tags, labels)
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}},suffix=${{ steps.build_features.outputs.image_suffix }}
            type=semver,pattern={{major}}.{{minor}},suffix=${{ steps.build_features.outputs.image_suffix }}
            type=semver,pattern={{major}},suffix=${{ steps.build_features.outputs.image_suffix }}
            type=raw,value=${{ steps.build_features.outputs.manual_version }}${{ steps.build_features.outputs.image_suffix }},enable=${{ github.event_name == 'workflow_dispatch' && steps.build_features.outputs.manual_version != '' }}
            type=sha,prefix=
            type=raw,value=main,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
            type=raw,value=edge,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
            type=raw,value=latest,enable=${{ github.event_name != 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/v') }}

      - name: Build and push Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: ${{ vars.TC780_CUTOVER_READY == 'true' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: CARGO_FEATURES=${{ steps.build_features.outputs.cargo_features }}
          cache-from: type=gha,scope=node
          cache-to: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && 'type=gha,mode=max,scope=node' || '' }}

  build-dstack:
    runs-on: ubuntu-latest
    needs: [version-guard]
    if: vars.TC780_CUTOVER_READY == 'true' && inputs.deploy_image_digest == '' && !inputs.rollback_audit_only && (github.event_name == 'release' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref == 'refs/heads/main'))
    outputs:
      digest: ${{ steps.build.outputs.digest }}
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Resolve optional build features
        id: build_features
        run: |
          FEATURES="dstack"
          IMAGE_SUFFIX=""
          SHA_PREFIX="dstack-"
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then
            FEATURES="dstack duckdb"
            IMAGE_SUFFIX="-duckdb"
            SHA_PREFIX="dstack-duckdb-"
          fi
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_tc_bench_v1 }}" = "true" ]; then
            FEATURES="$FEATURES tc-bench-v1"
            IMAGE_SUFFIX="${IMAGE_SUFFIX}-tc-bench-v1"
            SHA_PREFIX="${SHA_PREFIX}tc-bench-v1-"
          fi
          {
            echo "cargo_features=${FEATURES}"
            echo "image_suffix=${IMAGE_SUFFIX}"
            echo "sha_prefix=${SHA_PREFIX}"
            echo "manual_version=${{ inputs.image_version }}"
          } >> "$GITHUB_OUTPUT"

      - name: Extract metadata (tags, labels)
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}},suffix=-dstack${{ steps.build_features.outputs.image_suffix }}
            type=semver,pattern={{major}}.{{minor}},suffix=-dstack${{ steps.build_features.outputs.image_suffix }}
            type=semver,pattern={{major}},suffix=-dstack${{ steps.build_features.outputs.image_suffix }}
            type=raw,value=${{ steps.build_features.outputs.manual_version }}-dstack${{ steps.build_features.outputs.image_suffix }},enable=${{ github.event_name == 'workflow_dispatch' && steps.build_features.outputs.manual_version != '' }}
            type=sha,prefix=${{ steps.build_features.outputs.sha_prefix }}
            type=raw,value=dstack-main,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
            type=raw,value=dstack-edge,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
            type=raw,value=dstack,enable=${{ github.event_name != 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/v') }}

      - name: Build and push dstack Docker image
        id: build
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: CARGO_FEATURES=${{ steps.build_features.outputs.cargo_features }}
          cache-from: type=gha,scope=dstack
          cache-to: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && 'type=gha,mode=max,scope=dstack' || '' }}

  build-ingress:
    name: Build dstack ingress image
    runs-on: ubuntu-latest
    if: >-
      github.event_name == 'pull_request' ||
      (vars.TC780_CUTOVER_READY == 'true' && inputs.deploy_image_digest == '' && !inputs.rollback_audit_only && (github.event_name == 'release' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref == 'refs/heads/main')))
    outputs:
      digest: ${{ steps.build.outputs.digest }}
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v4

      # TC-970: no ticket, identifier or client address in the ingress logs;
      # TC-910 header buffers and the 32 MB body limit still hold. Runs before
      # the push so a failing wrapper is never published.
      - name: Test redacted ingress logging
        run: deploy/ingress/test-redacted-logging.sh

      - uses: docker/setup-buildx-action@v3

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels)
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/tinycloudlabs/tinycloud-node-ingress
          tags: |
            type=sha,prefix=
            type=raw,value=latest,enable={{is_default_branch}}

      - name: Build and push ingress image
        id: build
        uses: docker/build-push-action@v5
        with:
          context: deploy/ingress
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha,scope=ingress
          cache-to: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && 'type=gha,mode=max,scope=ingress' || '' }}
      - name: Record ingress image digest
        if: steps.build.outputs.digest != ''
        env:
          IMAGE_DIGEST: ${{ steps.build.outputs.digest }}
        run: echo "ghcr.io/tinycloudlabs/tinycloud-node-ingress@${IMAGE_DIGEST}" >> "$GITHUB_STEP_SUMMARY"
  deploy-phala:
    runs-on: ubuntu-latest
    needs: [version-guard, build, build-dstack]
    # TC-971: production-environment policy admits main and v* tags. Most deploy
    # secrets live there; PROD_TINYCLOUD_ADMIN_SECRET is repository-scoped.
    # Digest promotion additionally requires main, not a release tag.
    environment: production
    if: >-
      ${{ vars.TC780_CUTOVER_READY == 'true' && !cancelled() && !inputs.rollback_audit_only && needs.version-guard.result == 'success' &&
        github.event_name == 'workflow_dispatch' && inputs.deploy_phala &&
        ((inputs.deploy_image_digest != '' && github.ref == 'refs/heads/main' && needs.build.result == 'skipped' && needs.build-dstack.result == 'skipped') ||
          (inputs.deploy_image_digest == '' && needs.build.result == 'success' && needs.build-dstack.result == 'success')) }}
    outputs:
      image: ${{ steps.pin.outputs.image }}
      version: ${{ steps.pin.outputs.version }}
      revision: ${{ steps.pin.outputs.revision }}
      image_label: ${{ steps.pin.outputs.image_label }}
    permissions:
      contents: read
      packages: read
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # TC-940: production is built and promoted only from main. Refuse any
      # other dispatched ref before this job touches a deploy secret.
      - name: Guard main-only deploy source
        env:
          ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }}
        run: |
          set -euo pipefail
          git fetch --quiet --no-tags origin '+refs/heads/main:refs/remotes/origin/main'
          bash scripts/ci/check-deploy-ref.sh

      - name: Log in to GHCR for digest validation
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      # Pinned deliberately. phala 1.1.20 (published 2026-07-26) fails against
      # the existing centralized-KMS prod CVM with "CVM encrypted_env_pubkey is
      # required for centralized KMS", which broke the 2026-07-28 deploy. The
      # last green deploy (run 29953848846, 2026-07-22) ran on 1.1.19. Only
      # unpin once the prod CVM is migrated or 1.1.20+ is verified against it.
      - name: Install Phala CLI
        run: npm install -g phala@1.1.19

      - name: Install Phala env synchronization SDK
        # Same SDK generation as phala 1.1.19, isolated from the checkout.
        # Install before any sealed values enter the step environment.
        run: npm install --prefix "$RUNNER_TEMP/phala-env-sync" --no-save --ignore-scripts @phala/cloud@0.2.9

      - name: Guard production ancestry
        env:
          PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }}
          ALLOW_NON_DESCENDANT: ${{ inputs.allow_non_descendant }}
        run: |
          set -euo pipefail

          if [ "${ALLOW_NON_DESCENDANT}" = "true" ]; then
            echo "::warning::allow_non_descendant=true; skipping the production revision ancestry check"
            exit 0
          fi

          if ! CVM_JSON="$(mktemp)"; then
            echo "::error::Cannot create a temporary file for Phala's current compose"
            exit 1
          fi
          trap 'rm -f "${CVM_JSON}"' EXIT
          if ! phala cvms get tinycloud-node --json >"${CVM_JSON}" 2>/dev/null; then
            echo "::error::Cannot determine the running production revision from Phala"
            exit 1
          fi

          if ! RUNNING_IMAGE="$(python3 - "${CVM_JSON}" <<'PY'
          import json
          import re
          import sys
          from pathlib import Path

          try:
              response = json.loads(Path(sys.argv[1]).read_text())
          except (OSError, ValueError):
              sys.exit(0)

          if isinstance(response, str):
              compose = response
          elif isinstance(response, dict):
              compose_file = response.get("compose_file")
              if isinstance(compose_file, dict):
                  compose = compose_file.get("docker_compose_file", "")
              else:
                  compose = response.get("docker_compose_file", "")
          else:
              compose = ""
          if not isinstance(compose, str):
              sys.exit(0)

          images = re.findall(
              r"(?m)^\s*image:\s*['\"]?(ghcr\.io/tinycloudlabs/tinycloud-node@sha256:[0-9a-f]{64})",
              compose,
          )
          if len(images) == 1:
              print(images[0])
          PY
          )"; then
            echo "::error::Cannot determine a unique pinned production image from Phala's current compose"
            exit 1
          fi
          if [ -z "${RUNNING_IMAGE}" ]; then
            echo "::error::Cannot determine a unique pinned production image from Phala's current compose"
            exit 1
          fi

          if ! docker pull --platform linux/amd64 "${RUNNING_IMAGE}" >/dev/null 2>&1; then
            echo "::error::Cannot inspect the running production image in GHCR"
            exit 1
          fi
          if ! RUNNING_REVISION="$(docker image inspect "${RUNNING_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' 2>/dev/null)"; then
            echo "::error::Cannot determine the running production revision from its image label"
            exit 1
          fi
          if ! [[ "${RUNNING_REVISION}" =~ ^[0-9a-f]{40}$ ]] \
            || ! git cat-file -e "${RUNNING_REVISION}^{commit}" 2>/dev/null; then
            echo "::error::Cannot determine the running production revision from its image label"
            exit 1
          fi
          if ! git merge-base --is-ancestor "${RUNNING_REVISION}" HEAD; then
            echo "::error::Running production revision ${RUNNING_REVISION} is not an ancestor of ${GITHUB_SHA}; refusing a non-descendant deployment"
            exit 1
          fi
          echo "Running production revision ${RUNNING_REVISION} is an ancestor of ${GITHUB_SHA}."

      - name: Preflight quota authority
        env:
          QUOTA_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }}
          # Billing's storage-comp guard checks the same shared node/billing
          # admin value (billing TINYCLOUD_ADMIN_SECRET); see docs/releasing.md.
          QUOTA_BILLING_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }}
          QUOTA_BILLING_URL: ${{ secrets.PROD_TINYCLOUD_QUOTA_URL }}
        run: |
          set -euo pipefail
          set +x
          python3 scripts/ci/reconcile-quota.py --preflight

      - name: Validate and pin deployment image
        id: pin
        env:
          INPUT_DEPLOY_DIGEST: ${{ inputs.deploy_image_digest }}
          BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }}
          DEPLOY_VERSION: ${{ needs.version-guard.outputs.image_version }}
          INCLUDE_DUCKDB: ${{ inputs.include_duckdb }}
          INCLUDE_TC_BENCH_V1: ${{ inputs.include_tc_bench_v1 }}
          ROLLBACK_VERSION: ${{ inputs.rollback_image_version }}
          ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }}
          ROLLBACK_PRE_N0: ${{ inputs.rollback_pre_n0 }}
          ROLLBACK_PRIVACY_MODE: ${{ inputs.rollback_privacy_mode }}
          APPROVED_ROLLBACK_PRIVACY_MODE: ${{ vars.TC959_ROLLBACK_PRIVACY_MODE }}
        run: |
          set -euo pipefail
          DIGEST="${INPUT_DEPLOY_DIGEST:-${BUILD_DIGEST}}"
          [[ "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "::error::Invalid deployment digest"; exit 1; }
          export RESOLVED_IMAGE="${REGISTRY}/${IMAGE_NAME}@${DIGEST}"
          docker pull --platform linux/amd64 "${RESOLVED_IMAGE}"
          REVISION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
          VERSION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.version" }}')"
          EXPECTED="${DEPLOY_VERSION}-dstack"
          if [ "${INCLUDE_DUCKDB}" = "true" ]; then EXPECTED="${EXPECTED}-duckdb"; fi
          if [ "${INCLUDE_TC_BENCH_V1}" = "true" ]; then EXPECTED="${EXPECTED}-tc-bench-v1"; fi
          EXPECTED_REVISION="$(git rev-parse HEAD)"
          if [ -n "${ROLLBACK_VERSION}" ]; then
            EXPECTED_REVISION="${ROLLBACK_REVISION}"
          fi
          if [ "${REVISION}" != "${EXPECTED_REVISION}" ] || [ "${VERSION}" != "${EXPECTED}" ]; then
            echo "::error::Image revision/version labels do not match the selected checkout and requested features"
            exit 1
          fi
          python3 - <<'PY'
          import hashlib
          import os
          import re
          from pathlib import Path
          def read_artifact(relative):
              path = Path(relative)
              if path.is_symlink() or path.resolve() != Path.cwd().resolve() / relative:
                  raise SystemExit('::error::Deployment artifact path is not the fixed reviewed path')
              try:
                  return path.read_bytes()
              except OSError:
                  raise SystemExit('::error::Reviewed deployment artifact is unavailable') from None

          path = Path('docker-compose.dstack-postgres.yaml')
          if path.is_symlink() or path.resolve() != Path.cwd().resolve() / path:
              raise SystemExit('::error::Deployment destination is not the fixed reviewed path')
          native = 'ghcr.io/tinycloudlabs/tinycloud-node'
          reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f'
          n0_hash = 'ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6'
          pre_n0_hash = 'df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91'
          pre_n0_template_hash = '28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93'
          rollback = os.environ.get('ROLLBACK_PRE_N0') == 'true'
          mode = os.environ.get('ROLLBACK_PRIVACY_MODE') or 'unselected'
          if rollback:
              raw = read_artifact('ops/rollback/pre-n0-compose.yaml')
              if os.environ['RESOLVED_IMAGE'] != reviewed or hashlib.sha256(raw).hexdigest() != pre_n0_hash:
                  raise SystemExit('::error::Captured pre-N0 image/compose pin mismatch; deployment not attempted')
              if mode not in ('keep-json-file', 'stop-future-local-logs') or mode != os.environ.get('APPROVED_ROLLBACK_PRIVACY_MODE'):
                  raise SystemExit('::error::Rollback privacy decision is missing or mismatched')
              if mode == 'stop-future-local-logs':
                  raw = read_artifact('ops/rollback/pre-n0-stop-local-logs.yaml')
                  if hashlib.sha256(raw).hexdigest() != '315ceca54bf8bfefe2b395b4097fe38ed8ec847fa0688bc873220bd65efc99ec':
                      raise SystemExit('::error::Reviewed stop-future-logs artifact pin mismatch')
              n0_configuration = False
          else:
              if mode != 'unselected':
                  raise SystemExit('::error::Rollback privacy cannot modify a forward/current-compose deployment')
              raw = read_artifact(str(path))
              digest = hashlib.sha256(raw).hexdigest()
              n0_configuration = digest == n0_hash
              if n0_configuration and os.environ['RESOLVED_IMAGE'] != reviewed:
                  raise SystemExit('::error::Reviewed N0 compose cannot substitute a different image')
              if os.environ['RESOLVED_IMAGE'] == reviewed and digest not in (n0_hash, pre_n0_hash, pre_n0_template_hash):
                  raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose')
          content = raw.decode('utf-8')
          images = [
              image for image in re.finditer(r'(?m)^([ \t]+image:[ \t]+)([^\n]+)$', content)
              if image[2].strip().strip('"\'').startswith((native + ':', native + '@'))
          ]
          if len(images) != 1 or images[0][2].strip().strip('"\'') not in (native + ':dstack', reviewed):
              raise SystemExit('Expected exactly one :dstack or reviewed digest-pinned native image')
          image = images[0]
          if image[2].strip().strip('"\'') == reviewed and reviewed != os.environ['RESOLVED_IMAGE']:
              raise SystemExit('Reviewed compose digest does not match the selected deployment image')
          path.write_bytes((content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8'))
          with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
              output.write(f'n0_configuration_verified={str(n0_configuration).lower()}\n')
              output.write(f'rollback_pre_n0_verified={str(rollback).lower()}\n')
          if rollback:
              message = f'Rollback privacy: {mode}; no deletion of existing log files is requested.'
              print(message)
              with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as summary:
                  summary.write(message + '\n')
          PY
          {
            echo "image=${RESOLVED_IMAGE}"
            echo "version=${VERSION%%-dstack*}"
            echo "revision=${REVISION}"
            echo "image_label=${VERSION}"
          } >> "$GITHUB_OUTPUT"
          echo "Resolved image: ${RESOLVED_IMAGE} (revision ${REVISION}, version ${VERSION})"
          printf 'Deployment image: `%s` — version `%s`, revision `%s`.\n' \
            "${RESOLVED_IMAGE}" "${VERSION}" "${REVISION}" >> "$GITHUB_STEP_SUMMARY"

      # TC-500: Phala considers a CVM ready before compose services have
      # started. Validate the production-sealed bundle using the exact image
      # selected above against the production database/TLS/key configuration,
      # without opening a database, resolving dstack, network, or socket
      # access, before the deploy step can replace the healthy CVM.
      - name: Preflight sealed runtime configuration
        env:
          TINYCLOUD_STORAGE__DATABASE: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }}
          TINYCLOUD_DATABASE__WRITE_FENCE: ${{ inputs.sql_identity_fence }}
          TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: ${{ secrets.PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64 }}
          TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full
          TINYCLOUD_KEYS__TYPE: Dstack
        run: |
          set -euo pipefail
          IMAGE="$(sed -n -E 's/^    image: (ghcr.io\/tinycloudlabs\/tinycloud-node(:[^[:space:]]+|@sha256:[0-9a-f]{64}))$/\1/p' docker-compose.dstack-postgres.yaml)"
          if [ "$(printf '%s\n' "${IMAGE}" | awk 'NF { count += 1 } END { print count + 0 }')" != "1" ]; then
            echo "::error::Expected exactly one resolved TinyCloud Node image for runtime preflight"
            exit 1
          fi
          docker run --rm --network none --read-only --cap-drop ALL \
            --security-opt no-new-privileges \
            -e TINYCLOUD_STORAGE__DATABASE \
            -e TINYCLOUD_DATABASE__WRITE_FENCE \
            -e TINYCLOUD_SHARE_EMAIL__ENABLED=true \
            -e TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64 \
            -e TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE \
            -e TINYCLOUD_KEYS__TYPE \
            "${IMAGE}" --validate-config

      # TC-959: only env_keys/encrypted_env may change here. Never copy the
      # precedent's raw-response logs or its unguarded compose_hash retry.
      # https://docs.phala.com/phala-cloud/references/cloud-js-sdk/update-cvm-envs
      - name: Prepare allowed_envs synchronization
        run: |
          set -euo pipefail
          cat > "$RUNNER_TEMP/phala-env-sync/sync.mjs" <<'JS'
          import { readFileSync } from "node:fs";
          import { setTimeout as sleep } from "node:timers/promises";

          // SDK DEBUG can print request bodies and server-supplied errors.
          delete process.env.DEBUG;
          let phase = "sealed env validation";
          const requireCondition = (condition) => {
            if (!condition) throw new Error("guard failed");
          };
          const validName = (name) => typeof name === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(name);
          const namesFrom = (compose) => {
            const names = compose?.allowed_envs;
            requireCondition(Array.isArray(names) && names.every(validName));
            requireCondition(new Set(names).size === names.length);
            return names;
          };

          try {
            const {
              createClient, getCvmComposeFile, getCvmInfo, updateCvmEnvs,
              encryptEnvVars, parseEnvVars,
            } = await import("@phala/cloud");
            requireCondition(Boolean(process.env.PHALA_CLOUD_API_KEY));
            const source = readFileSync(".phala-prod.env", "utf8");
            const envs = [];
            const keys = new Set();
            for (const line of source.split("\n")) {
              if (line === "") continue;
              const separator = line.indexOf("=");
              const key = line.slice(0, separator);
              const value = line.slice(separator + 1);
              requireCondition(separator > 0 && validName(key) && !keys.has(key));
              requireCondition(value.length > 0 && !/[\r\n]/.test(value));
              keys.add(key);
              envs.push({ key, value });
            }
            requireCondition(keys.has("BETTERSTACK_SOURCE_TOKEN"));
            // Match the CLI's dotenv interpretation exactly; no truncated,
            // quoted or comment-stripped value may replace the intended one.
            requireCondition(JSON.stringify(parseEnvVars(source)) === JSON.stringify(envs));
            const client = createClient({
              apiKey: process.env.PHALA_CLOUD_API_KEY,
              baseURL: "https://cloud-api.phala.com/api/v1",
              version: "2026-01-21",
              timeout: 15000,
              retry: 0,
            });
            const ref = { id: "tinycloud-node" };
            phase = "complete whitelist validation";
            const allowed = namesFrom(await getCvmComposeFile(client, ref));
            // Retaining a name cannot recover a value absent from this file.
            // Check before deploy AND again immediately before any env PATCH.
            requireCondition(allowed.every((key) => keys.has(key)));
            const missing = [...keys].filter((key) => !allowed.includes(key));
            if (process.argv.includes("--check")) {
              console.log(`Validated complete sealed set: ${keys.size} env names.`);
            } else if (missing.length === 0) {
              console.log(`allowed_envs verified: ${keys.size} env names; no PATCH needed.`);
            } else {
              phase = "public encryption key validation";
              const info = await getCvmInfo(client, ref);
              const pubkey = info.kms_info?.encrypted_env_pubkey ?? info.encrypted_env_pubkey;
              requireCondition(typeof pubkey === "string" && /^(0x)?[0-9a-fA-F]{64}$/.test(pubkey));
              const encrypted = await encryptEnvVars(envs, pubkey);
              phase = "env whitelist update";
              const result = await updateCvmEnvs(client, {
                ...ref, encrypted_env: encrypted, env_keys: [...allowed, ...missing],
              });
              // An on-chain precondition needs a separately reviewed
              // addComposeHash transaction. Never assume hash-only approval.
              requireCondition(result.status === "in_progress" && typeof result.correlation_id === "string" && result.correlation_id.length > 0);
              console.log(`allowed_envs update accepted: ${missing.length} added names.`);
              phase = "env whitelist convergence";
              const deadline = Date.now() + 600000;
              let completed = false;
              let observedOperation = false;
              const pending = new Set(["starting", "stopping", "stopped", "restarting", "upgrading", "updating", "creating"]);
              while (Date.now() < deadline) {
                await sleep(5000);
                const current = await getCvmInfo(client, ref);
                requireCondition(current.status === "running" || pending.has(current.status));
                const after = namesFrom(await getCvmComposeFile(client, ref));
                const progress = current.progress;
                // Prefer this operation's correlation. If optional progress
                // is absent, require a post-PATCH non-running/running cycle.
                if (progress?.correlation_id === result.correlation_id ||
                    (!progress?.correlation_id && pending.has(current.status))) observedOperation = true;
                // Missing/null progress can mean a fast or still-queued
                // restart. Stored names + running alone are not completion.
                const settled = progress == null || (
                  progress && !progress.target && !progress.started_at && !progress.correlation_id
                );
                if (observedOperation && current.status === "running" && settled && [...keys].every((key) => after.includes(key))) {
                  completed = true;
                  break;
                }
              }
              requireCondition(completed);
              console.log(`allowed_envs verified after update: ${keys.size} env names.`);
            }
          } catch {
            // Neither response message/JSON, ciphertext, nor exception detail
            // is safe for public Actions logs.
            console.error(`::error::allowed_envs synchronization failed during ${phase}; response details withheld.`);
            process.exitCode = 1;
          }
          JS

      - name: Deploy to Phala Cloud
        id: deploy
        env:
          PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }}
          PHALA_ENV_DATABASE_URL: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }}
          PHALA_ENV_S3_BUCKET: ${{ secrets.PROD_TINYCLOUD_S3_BUCKET }}
          PHALA_ENV_S3_ENDPOINT: ${{ secrets.PROD_TINYCLOUD_S3_ENDPOINT }}
          PHALA_ENV_AWS_KEY: ${{ secrets.PROD_TINYCLOUD_AWS_ACCESS_KEY_ID }}
          PHALA_ENV_AWS_SECRET: ${{ secrets.PROD_TINYCLOUD_AWS_SECRET_ACCESS_KEY }}
          PHALA_ENV_AWS_REGION: ${{ secrets.PROD_TINYCLOUD_AWS_REGION }}
          PHALA_ENV_TINYCLOUD_QUOTA_URL: ${{ secrets.PROD_TINYCLOUD_QUOTA_URL }}
          PHALA_ENV_TINYCLOUD_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }}
          PHALA_ENV_SHARE_TRUST_BUNDLE_BASE64: ${{ secrets.PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64 }}
          PHALA_ENV_CLOUDFLARE_API_TOKEN: ${{ secrets.PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN }}
          PHALA_ENV_DSTACK_GATEWAY_DOMAIN: ${{ secrets.PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN }}
          PHALA_ENV_CERTBOT_EMAIL: ${{ secrets.PROD_TINYCLOUD_CERTBOT_EMAIL }}
          PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }}
          PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }}
        # Update the existing prod CVM (name: tinycloud-node) in the
        # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is
        # workspace-scoped and must be issued from that workspace.
        run: |
          set -euo pipefail
          set +x
          umask 077
          trap 'rm -f .phala-prod.env' EXIT

          required_vars=(
            PHALA_CLOUD_API_KEY
            PHALA_ENV_DATABASE_URL
            PHALA_ENV_S3_BUCKET
            PHALA_ENV_S3_ENDPOINT
            PHALA_ENV_AWS_KEY
            PHALA_ENV_AWS_SECRET
            PHALA_ENV_AWS_REGION
            PHALA_ENV_TINYCLOUD_QUOTA_URL
            PHALA_ENV_TINYCLOUD_ADMIN_SECRET
            PHALA_ENV_SHARE_TRUST_BUNDLE_BASE64
            PHALA_ENV_CLOUDFLARE_API_TOKEN
            PHALA_ENV_DSTACK_GATEWAY_DOMAIN
            PHALA_ENV_CERTBOT_EMAIL
            PHALA_ENV_BETTERSTACK_SOURCE_TOKEN
            PHALA_ENV_SQL_IDENTITY_FENCE
          )

          for var in "${required_vars[@]}"; do
            if [ -z "${!var:-}" ]; then
              echo "::error::Missing required GitHub secret for Phala deploy: ${var#PHALA_ENV_}"
              exit 1
            fi
            if [[ "${!var}" == *$'\n'* || "${!var}" == *$'\r'* ]]; then
              echo "::error::Phala deploy inputs must be single-line values"
              exit 1
            fi
          done

          # Also restrict a file left by an earlier failed runner step before
          # writing any value; umask alone does not tighten an existing file.
          : > .phala-prod.env
          chmod 0600 .phala-prod.env
          {
            printf 'DATABASE_URL=%s\n' "$PHALA_ENV_DATABASE_URL"
            printf 'S3_BUCKET=%s\n' "$PHALA_ENV_S3_BUCKET"
            printf 'S3_ENDPOINT=%s\n' "$PHALA_ENV_S3_ENDPOINT"
            printf 'AWS_KEY=%s\n' "$PHALA_ENV_AWS_KEY"
            printf 'AWS_SECRET=%s\n' "$PHALA_ENV_AWS_SECRET"
            printf 'AWS_REGION=%s\n' "$PHALA_ENV_AWS_REGION"
            printf 'TINYCLOUD_QUOTA_URL=%s\n' "$PHALA_ENV_TINYCLOUD_QUOTA_URL"
            printf 'TINYCLOUD_ADMIN_SECRET=%s\n' "$PHALA_ENV_TINYCLOUD_ADMIN_SECRET"
            printf 'SHARE_TRUST_BUNDLE_BASE64=%s\n' "$PHALA_ENV_SHARE_TRUST_BUNDLE_BASE64"
            printf 'CLOUDFLARE_API_TOKEN=%s\n' "$PHALA_ENV_CLOUDFLARE_API_TOKEN"
            printf 'DSTACK_GATEWAY_DOMAIN=%s\n' "$PHALA_ENV_DSTACK_GATEWAY_DOMAIN"
            printf 'CERTBOT_EMAIL=%s\n' "$PHALA_ENV_CERTBOT_EMAIL"
            printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN"
            printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE"
            # SSH public keys authorized on the CVM (checked into the repo,
            # see dstack-authorized-keys). The Phala Cloud pre-launch script
            # writes DSTACK_AUTHORIZED_KEYS to /home/root/.ssh/authorized_keys
            # at boot, enabling `phala ssh tinycloud-node`.
            printf 'DSTACK_AUTHORIZED_KEYS=%s\n' "$(cat dstack-authorized-keys)"
          } > .phala-prod.env

          # Validate complete env names/whitelist before any CVM mutation.
          node "$RUNNER_TEMP/phala-env-sync/sync.mjs" --check

          # Fail before replacing the running CVM if a required compose input
          # is absent. Phala's control-plane ready state does not prove that
          # docker compose interpolation or container startup succeeded.
          if ! docker compose --env-file .phala-prod.env \
            -f docker-compose.dstack-postgres.yaml config --quiet >/dev/null 2>&1; then
            echo "::error::Sealed compose validation failed; deployment not attempted"
            exit 1
          fi

          # The CLI/SDK can include request details in errors. Never forward
          # those responses into Actions logs, even when deployment fails.
          # A failing CLI/update can still have restarted the CVM.
          echo "cvm_mutation_attempted=true" >> "$GITHUB_OUTPUT"
          if ! phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait >/dev/null 2>&1; then
            echo "::error::Phala deployment failed; inspect the private control plane"
            exit 1
          fi

          # Do not assume -e registered the whitelist; verify/sync it explicitly.
          node "$RUNNER_TEMP/phala-env-sync/sync.mjs"

      - name: Remove sealed deployment environment
        if: always()
        run: rm -f .phala-prod.env

      # TC-312: prove the deploy actually converged. The image tag matching the
      # crate version is enforced at build time by version-guard, but nothing
      # previously confirmed the running CVM serves it — the incident where prod
      # reported a stale version went unnoticed for exactly this reason. Poll the
      # public /version endpoint and fail the job if it never reports the tag we
      # just deployed.
      - name: Verify deployed version
        env:
          EXPECTED: ${{ steps.pin.outputs.version }}
          IMAGE_REVISION: ${{ steps.pin.outputs.revision }}
        run: |
          set -euo pipefail

          # Only validated image labels describe the binary, not Cargo.toml
          # at the workflow checkout when an existing digest is promoted.
          echo "Expecting node.tinycloud.xyz to report version ${EXPECTED} (revision ${IMAGE_REVISION})"

          # The CVM needs a short while to pull the new image and restart, so
          # retry with a fixed backoff (~5 min total) before giving up.
          ATTEMPTS=30
          SLEEP=10
          REPORTED=""
          for i in $(seq 1 "${ATTEMPTS}"); do
            REPORTED="$(curl -fsS --max-time 15 https://node.tinycloud.xyz/version | jq -r '.version' || true)"
            echo "attempt ${i}/${ATTEMPTS}: /version reported '${REPORTED}'"
            if [ "${REPORTED}" = "${EXPECTED}" ]; then
              echo "node.tinycloud.xyz is serving ${EXPECTED}."
              exit 0
            fi
            sleep "${SLEEP}"
          done

          echo "::error::node.tinycloud.xyz still reports '${REPORTED}' after $((ATTEMPTS * SLEEP))s, expected '${EXPECTED}'. The deploy did not converge."
          exit 1

      # Ingress TLS readiness is required health. Restore quotas before the
      # separate Policy probes, which still fail the deployment if invalid.
      - name: Verify deployed ingress health
        env:
          EXPECTED: ${{ steps.pin.outputs.version }}
        run: |
          set -euo pipefail
          for attempt in $(seq 1 30); do
            REPORTED="$(curl -fsS --max-time 10 https://tee.node.tinycloud.xyz/version | jq -r '.version' || true)"
            if [ "${REPORTED}" = "${EXPECTED}" ]; then
              echo "Direct TEE ingress is serving ${EXPECTED}."
              exit 0
            fi
            sleep 5
          done
          echo "::error::Direct TEE ingress did not become healthy at version ${EXPECTED}."
          exit 1

      # General reviewed authority/values, freshly authorized for every deploy.
      # Same job, after the final env operation and required health, before
      # ancillary Policy probes. Billing mode is verification-only.

      - name: Restore storage quota overrides
        id: quotas
        timeout-minutes: 5
        if: ${{ success() && inputs.authorize_quota_values }}
        env:
          # Repository-scoped; never read by an agent or passed in argv.
          QUOTA_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }}
          QUOTA_BILLING_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }}
          QUOTA_BILLING_URL: ${{ secrets.PROD_TINYCLOUD_QUOTA_URL }}
        run: |
          set -euo pipefail
          set +x
          python3 scripts/ci/reconcile-quota.py

      # TC-465: Share requires the complete Policy/v3 control plane before it
      # can create or receive an exact-email share. A version-only check let a
      # pre-TC-405 image look healthy while every required route returned 404.
      # Empty JSON is deliberately rejected before any handler can mutate
      # policy state, so this proves route mounting against the direct TEE
      # origin without creating a binding, challenge, policy, or delegation.
      - name: Verify deployed Policy/v3 routes
        run: |
          set -euo pipefail

          NODE_ORIGIN="https://tee.node.tinycloud.xyz"
          ROUTES=(
            /policy/v3/enforcer-bindings
            /policy/v3/policies
            /policy/v3/challenges
            /policy/v3/delegations
            /policy/v3/deliveries/authorize
          )

          for route in "${ROUTES[@]}"; do
            RESPONSE_FILE="$(mktemp)"
            STATUS="$(curl -sS --max-time 15 -o "${RESPONSE_FILE}" -w '%{http_code}' \
              -X POST \
              -H 'accept: application/json' \
              -H 'content-type: application/json' \
              --data '{}' \
              "${NODE_ORIGIN}${route}")"

            # Delivery is authenticated before request-body validation, so its
            # intentionally unsigned probe must be exactly 401. Other
            # control-plane routes reject the body first (400 or 422). Neither
            # case may silently accept a missing route or a server failure.
            if [ "${route}" = "/policy/v3/deliveries/authorize" ]; then
              EXPECTED="401"
              case "${STATUS}" in 401) VALID=true ;; *) VALID=false ;; esac
            else
              EXPECTED="400 or 422"
              case "${STATUS}" in 400|422) VALID=true ;; *) VALID=false ;; esac
            fi

            if [ "${VALID}" != true ]; then
              echo "::error::${route} returned HTTP ${STATUS}; expected ${EXPECTED} for an intentionally invalid unauthenticated request."
              head -c 1000 "${RESPONSE_FILE}" || true
              rm -f "${RESPONSE_FILE}"
              exit 1
            fi

            echo "${route}: mounted (HTTP ${STATUS})"
            rm -f "${RESPONSE_FILE}"
          done

      - name: Report unverified storage overrides
        if: >-
          ${{ always() && steps.deploy.outputs.cvm_mutation_attempted == 'true' && steps.quotas.outcome != 'success' }}
        run: |
          set -euo pipefail
          MESSAGE="OVERRIDES UNVERIFIED: a CVM mutation was attempted without successful authorized quota reconciliation. Authorized operator must resolve before another dispatch."
          echo "::error::${MESSAGE}"
          printf '%s\n' "${MESSAGE}" >> "$GITHUB_STEP_SUMMARY"
          exit 1

  # TC-1176: the 2026-10-14 read hang began ~40 min after a promote. After any
  # Phala deploy attempt (including rollbacks and failed verifications, which
  # may still have changed prod), start the Write Probe's dense promote watch:
  # reads every 30 s for 5 min, then every 5 min for 60 min, paging on 2 slow
  # (>5 s) or failed reads in a row. A separate job so the production-secret
  # deploy job never holds `actions: write`.
  promote-watch:
    runs-on: ubuntu-latest
    needs: [deploy-phala]
    if: ${{ always() && needs.deploy-phala.result != 'skipped' && needs.deploy-phala.result != 'cancelled' }}
    timeout-minutes: 5
    permissions:
      actions: write
    steps:
      - name: Dispatch Write Probe promote watch
        env:
          GH_TOKEN: ${{ github.token }}
          IMAGE: ${{ needs.deploy-phala.outputs.image }}
          IMAGE_VERSION: ${{ needs.deploy-phala.outputs.version }}
          IMAGE_LABEL: ${{ needs.deploy-phala.outputs.image_label }}
          IMAGE_REVISION: ${{ needs.deploy-phala.outputs.revision }}
        run: |
          set -euo pipefail
          if [ -n "${IMAGE}" ]; then
            echo "Promote watch for ${IMAGE_LABEL} (version ${IMAGE_VERSION}) revision ${IMAGE_REVISION}: ${IMAGE}"
          else
            echo "Promote watch after deployment attempt; image validation did not complete."
          fi
          gh workflow run write-probe.yml -R "${GITHUB_REPOSITORY}" --ref main -f mode=dense