ci: promote a released main image with the reviewed N0 compose template

.github/workflows/docker.yml

@@ -594,6 +594,8 @@ jobs:
             exit 1
           fi
           echo "Running production revision ${RUNNING_REVISION} is an ancestor of ${GITHUB_SHA}."
+          # Later steps refuse a deployment older than what is running.
+          echo "RUNNING_REVISION=${RUNNING_REVISION}" >> "$GITHUB_ENV"
 
       - name: Preflight quota authority
         env:
@@ -639,10 +641,11 @@ jobs:
             echo "::error::Image revision/version labels do not match the selected checkout and requested features"
             exit 1
           fi
-          python3 - <<'PY'
+          LABEL_VERSION="${VERSION}" LABEL_REVISION="${REVISION}" python3 - <<'PY'
           import hashlib
           import os
           import re
+          import subprocess
           from pathlib import Path
           def read_artifact(relative):
               path = Path(relative)
@@ -653,12 +656,58 @@ jobs:
               except OSError:
                   raise SystemExit('::error::Reviewed deployment artifact is unavailable') from None
 
+          def git(*args):
+              result = subprocess.run(['git', *args], capture_output=True, text=True)
+              return result.stdout if result.returncode == 0 else ''
+
           path = Path('docker-compose.dstack-postgres.yaml')
           if path.is_symlink() or path.resolve() != Path.cwd().resolve() / path:
               raise SystemExit('::error::Deployment destination is not the fixed reviewed path')
           native = 'ghcr.io/tinycloudlabs/tinycloud-node'
           reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f'
-          n0_hash = 'c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7'
+          # TC-959: main's N0 compose (#349 c33c2302…) with the 95m ingress body
+          # cap (raw d95bfb70…), its single node digest normalized to 64 zeros.
+          n0_template_hash = '51f8e9e93edc0e16025d27ee42abe003ed5d1cf26ba8f437d5120894aca64bdb'
+          node_pin = re.compile(rb'(?m)^    image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$')
+          def n0_template(raw):
+              pins = list(node_pin.finditer(raw))
+              if len(pins) != 1:
+                  return False
+              normalized = raw[:pins[0].start(1)] + b'0' * 64 + raw[pins[0].end(1):]
+              return hashlib.sha256(normalized).hexdigest() == n0_template_hash
+
+          def require_release():
+              # Only a recorded, released main image may replace the N0 pin:
+              # its labels, vX.Y.Z tag and crate version must all agree.
+              refuse = '::error::Reviewed N0 compose accepts only a recorded released main image; deployment not attempted'
+              label = re.fullmatch(r'([0-9]+\.[0-9]+\.[0-9]+)-dstack', os.environ['LABEL_VERSION'])
+              revision = os.environ['LABEL_REVISION']
+              if (not label or not re.fullmatch(r'[0-9a-f]{40}', revision) or not os.environ.get('INPUT_DEPLOY_DIGEST')
+                      or os.environ['RESOLVED_IMAGE'] != native + '@' + os.environ['INPUT_DEPLOY_DIGEST']
+                      or os.environ.get('ROLLBACK_VERSION') != label[1] or os.environ.get('ROLLBACK_REVISION') != revision):
+                  raise SystemExit(refuse)
+              if git('rev-parse', '--verify', '--quiet', f'refs/tags/v{label[1]}^{{commit}}').strip() != revision:
+                  raise SystemExit(refuse)
+              crate = re.search(r'(?m)^version = "([^"]*)"$', git('show', f'{revision}:tinycloud-node-server/Cargo.toml'))
+              if not crate or crate[1] != label[1]:
+                  raise SystemExit(refuse)
+              require_release_descends(revision)
+
+          def is_ancestor(older, newer):
+              return subprocess.run(['git', 'merge-base', '--is-ancestor', older, newer], capture_output=True).returncode == 0
+
+          running = os.environ.get('RUNNING_REVISION', '')
+
+          def require_not_older_than_running(revision):
+              # No cross-migration downgrade: the running production revision
+              # must be the deployed image's revision or its ancestor.
+              if running and not is_ancestor(running, revision):
+                  raise SystemExit('::error::Refusing to deploy an image older than the running production revision; use the captured pre-N0 rollback for an intentional rollback')
+
+          def require_release_descends(revision):
+              if not re.fullmatch(r'[0-9a-f]{40}', running) or running == revision or not is_ancestor(running, revision):
+                  raise SystemExit('::error::Released image must be a strict descendant of the running production revision; deployment not attempted')
+
           pre_n0_hash = 'df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91'
           pre_n0_template_hash = '28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93'
           rollback = os.environ.get('ROLLBACK_PRE_N0') == 'true'
@@ -679,22 +728,29 @@ jobs:
                   raise SystemExit('::error::Rollback privacy cannot modify a forward/current-compose deployment')
               raw = read_artifact(str(path))
               digest = hashlib.sha256(raw).hexdigest()
-              n0_configuration = digest == n0_hash
-              if n0_configuration and os.environ['RESOLVED_IMAGE'] != reviewed:
-                  raise SystemExit('::error::Reviewed N0 compose cannot substitute a different image')
-              if os.environ['RESOLVED_IMAGE'] == reviewed and digest not in (n0_hash, pre_n0_hash, pre_n0_template_hash):
-                  raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose')
+              n0_configuration = n0_template(raw)
+              if os.environ['RESOLVED_IMAGE'] == reviewed:
+                  if not n0_configuration and digest not in (pre_n0_hash, pre_n0_template_hash):
+                      raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose')
+                  require_not_older_than_running(os.environ['LABEL_REVISION'])
+              elif n0_configuration:
+                  require_release()
           content = raw.decode('utf-8')
           images = [
               image for image in re.finditer(r'(?m)^([ \t]+image:[ \t]+)([^\n]+)$', content)
               if image[2].strip().strip('"\'').startswith((native + ':', native + '@'))
           ]
-          if len(images) != 1 or images[0][2].strip().strip('"\'') not in (native + ':dstack', reviewed):
+          value = images[0][2].strip().strip('"\'') if len(images) == 1 else ''
+          if not n0_configuration and value not in (native + ':dstack', reviewed):
               raise SystemExit('Expected exactly one :dstack or reviewed digest-pinned native image')
-          image = images[0]
-          if image[2].strip().strip('"\'') == reviewed and reviewed != os.environ['RESOLVED_IMAGE']:
+          if not n0_configuration and value == reviewed and reviewed != os.environ['RESOLVED_IMAGE']:
               raise SystemExit('Reviewed compose digest does not match the selected deployment image')
-          path.write_bytes((content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8'))
+          image = images[0]
+          deployed = (content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8')
+          selected_digest = os.environ['RESOLVED_IMAGE'].rsplit(':', 1)[1].encode()
+          if n0_configuration and (not n0_template(deployed) or [pin[1] for pin in node_pin.finditer(deployed)] != [selected_digest]):
+              raise SystemExit('::error::Deployed N0 compose must differ from the reviewed template only by the selected digest')
+          path.write_bytes(deployed)
           with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
               output.write(f'n0_configuration_verified={str(n0_configuration).lower()}\n')
               output.write(f'rollback_pre_n0_verified={str(rollback).lower()}\n')

docker-compose.dstack-postgres.yaml

@@ -111,7 +111,7 @@ services:
       - CERTBOT_EMAIL=${CERTBOT_EMAIL}
       - SET_CAA=true
       - DNS_PROVIDER=cloudflare
-      - CLIENT_MAX_BODY_SIZE=32m
+      - CLIENT_MAX_BODY_SIZE=95m
     volumes:
       - /var/run/dstack.sock:/var/run/dstack.sock
       - cert-data:/etc/letsencrypt

docs/releasing.md

@@ -66,12 +66,14 @@ main's history (`scripts/ci/check-deploy-ref.sh`, before builds and again
 inside `deploy-phala`). Other refs fail `Deploy refused (main-only)`.
 
 **Release order:** approved #310 must never merge alone. Land #310 and its
-stacked recovery change atomically as one lineage, preserving emergency
-re-promotion and security rollback first. #309's sidecar compose belongs
-only to the later approved logging window, not this workflow/security
-release. The security card remains rollback-not-ready until the combined
-path is reviewed and landed. These docs grant no merge or dispatch authority.
-Merging either workflow change is a push, not a deployment dispatch.
+stacked recovery changes atomically as one lineage, preserving emergency
+re-promotion and security rollback first. #309's sidecar compose then ships
+on the unchanged 1.20.6 image under that lineage's exact raw-hash lock. The
+later main release lands this template lock and the 95m ingress body cap,
+and is promoted by one dispatch (see
+[Main release with N0 log shipping](#main-release-with-n0-log-shipping-one-dispatch)).
+These docs grant no merge or dispatch authority. Merging either workflow
+change is a push, not a deployment dispatch.
 
 ### The `production` environment (TC-971)
 
@@ -216,10 +218,13 @@ its branch policy before any workflow that names it runs.
    - `image_version` and recorded labels describe the immutable image,
      never an unrelated current crate version. The revision must be in
      main history. A future non-N0 digest uses this same contract.
-   - The reviewed N0 compose remains raw-hash locked to its exact image;
-     it cannot substitute a future security image. Before the later
-     logging window, the ordinary main `:dstack` compose supports forward
-     image promotions without installing the sidecar.
+   - The reviewed N0 compose is locked as a template: its raw bytes with
+     the single node digest normalized. Only a recorded *released* main
+     image may fill that pin; see
+     [Main release with N0 log shipping](#main-release-with-n0-log-shipping-one-dispatch).
+     Before #309 lands,
+     the ordinary main `:dstack` compose supports forward image
+     promotions without installing the sidecar.
    - `sql_identity_fence=false` is required. The input defaults to `true`,
      which re-fences SQL/DuckDB in production (503s).
    - The production ancestry guard still applies: the revision prod is running
@@ -277,6 +282,69 @@ its branch policy before any workflow that names it runs.
    `gh workflow run write-probe.yml -R TinyCloudLabs/tinycloud-node --ref main -f mode=dense`.
    Details: `test/probe/README.md`.
 
+### Main release with N0 log shipping (one dispatch)
+
+Ships a main release on #309's log sidecar compose. Before dispatching,
+`main` must contain the combined workflow lineage, #309, the template-lock
+change that also raises the ingress `CLIENT_MAX_BODY_SIZE` to `95m`, and the
+merged `chore(release): 1.21.0` commit tagged `v1.21.0`. Take the digest and
+labels of the tag build's `1.21.0-dstack` image as in step 1 above:
+
+```sh
+gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \
+  -f deploy_phala=true \
+  -f authorize_quota_values=true \
+  -f image_version=1.21.0 \
+  -f deploy_image_digest=sha256:<release-dstack-digest> \
+  -f rollback_image_version=1.21.0 \
+  -f rollback_image_revision=<release-commit-sha> \
+  -f sql_identity_fence=false
+```
+
+`rollback_image_version` and `rollback_image_revision` name the image being
+**deployed** (here the release), not an image to roll back to. Despite the
+input names, they are the recorded labels of `deploy_image_digest`.
+
+Before any CVM mutation the workflow requires:
+
+- the main compose, with its one node digest replaced by 64 zeros, to hash
+  to the reviewed N0 template
+  **`51f8e9e93edc0e16025d27ee42abe003ed5d1cf26ba8f437d5120894aca64bdb`**
+  (raw `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50` normalized: the #349 compose
+  `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`, which
+  keeps `TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"` and #346's Vector
+  config, with only `CLIENT_MAX_BODY_SIZE=32m` → `95m`). Ingress, socat,
+  Vector and the log profile stay byte-exact;
+- the deployed compose to differ from that template only by
+  `deploy_image_digest`;
+- OCI labels `1.21.0-dstack` / `<release-commit-sha>`, the `v1.21.0` tag on
+  that commit, its `Cargo.toml` at `1.21.0`, and the commit in main history.
+  An untagged, mislabelled, feature-variant or freshly built image is
+  refused;
+- the release revision to be a **strict descendant** of the revision running
+  in production (read from the CVM's pinned image by the ancestry guard).
+  A downgrade (for example 1.20.5 or 1.0.0), the same revision as running,
+  or an unknown running revision is refused. The same ancestry rule makes
+  the 12c same-image command refuse an image older than the running
+  revision once 1.21.0 is live, so it cannot downgrade across migrations.
+  The captured pre-N0 rollback is the only intentional downgrade route.
+
+Sealed preflight, `BETTERSTACK_SOURCE_TOKEN` sealing with the
+`allowed_envs` sync, and the general quota reconciliation run as on every
+deploy. The template's comment naming 1.20.6 is reviewed text, not image
+identity; `/version` and the labels report the release. Reproduce the
+template hash from any ref:
+
+```sh
+git show main:docker-compose.dstack-postgres.yaml \
+  | sed -E '/^    image: ghcr\.io\/tinycloudlabs\/tinycloud-node@sha256:/s/[0-9a-f]{64}$/0000000000000000000000000000000000000000000000000000000000000000/' \
+  | sha256sum
+```
+
+Rolling this release back means a database snapshot restore (a separate
+decision) plus the [captured pre-N0 rollback](#tc-1302-captured-pre-n0-rollback)
+to 1.20.6 with Sam's explicit privacy mode.
+
 ## Rolling back
 
 Keep the digest, actual OCI version and actual revision of every promoted
@@ -306,14 +374,13 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \
 ```
 
 The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**.
-The N0 raw lock **changed again in TC-1274**, which sets `TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"` in the production compose (whichever of this and #347 lands second re-pins). It previously changed in #346 (TC-959), which passes the validated N2 correlation fields through the log sidecar; the hash below is the TC-1274 lock:
-`c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`,
-from #346 final reviewed head.
-It supersedes the #309 lock `ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6`,
-from #309 reviewed head `44eeb20a1461ef66b5af492e3be4c42be2728b67`,
-[approval](https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050).
-Current pre-N0 tag/pinned equivalents are separately hash-checked; arbitrary
-same-image config substitutions fail before mutation.
+With the template lock on main this redeploys exactly main's raw compose
+`d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50`
+(the #349 compose `c33c2302…` plus the 95m ingress body cap), whatever digest
+main's template currently pins. It is not a rollback of a
+later release across a schema migration. Current pre-N0 tag/pinned
+equivalents are separately hash-checked; arbitrary same-image config
+substitutions fail before mutation.
 
 ### TC-1302 captured pre-N0 rollback
 

docs/runbooks/tc-959-n0-log-shipping.md

@@ -147,6 +147,7 @@ Keep these identities distinct:
 | Before: verifier `docker_compose_file` / hash | Raw preserved pre-N0 compose: `df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91` |
 | Before: application `compose_hash` | `bfe9028a58b23238188aa7d70b82282a7cd6414cd83dcc47a3b295c6f64fe2f8`; a distinct reported identity, not the raw-file hash |
 | After: current profile-enabled raw compose | `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`; bind this artifact to the final reviewed PR head before any promotion |
+| After the main release: template-locked raw compose | `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50` (main's compose with only `CLIENT_MAX_BODY_SIZE=32m` → `95m`); node digest normalized, template `51f8e9e93edc0e16025d27ee42abe003ed5d1cf26ba8f437d5120894aca64bdb`. See [`docs/releasing.md`](../releasing.md#main-release-with-n0-log-shipping-one-dispatch) |
 
 The raw after hash identifies the profile-enabled compose, not deployment approval. Record its final reviewed PR head and required CI evidence in the officer handoff. Do not infer a raw-file hash from the distinct application-reported compose hash.
 

scripts/ci/fixtures/tc959-n0-compose.yaml

@@ -111,7 +111,7 @@ services:
       - CERTBOT_EMAIL=${CERTBOT_EMAIL}
       - SET_CAA=true
       - DNS_PROVIDER=cloudflare
-      - CLIENT_MAX_BODY_SIZE=32m
+      - CLIENT_MAX_BODY_SIZE=95m
     volumes:
       - /var/run/dstack.sock:/var/run/dstack.sock
       - cert-data:/etc/letsencrypt