From 1b7c0dd00cb1614d9c01342c0d94b5619fe01ff3 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 02:41:54 +0000 Subject: [PATCH 1/3] ci: promote a released main image with the reviewed N0 compose template Lock #309's compose as a template (raw bytes with the single node digest normalized) so the main release and the sidecar ship in one dispatch. Only a recorded released image (labels, vX.Y.Z tag and crate version agree) may fill the pin; same-image recovery and the captured pre-N0 rollback are unchanged. Refs TC-959 --- .github/workflows/docker.yml | 59 +++++++++++++++---- docs/releasing.md | 87 ++++++++++++++++++++++------ scripts/ci/test-deploy-recovery.py | 92 +++++++++++++++++++++++++----- 3 files changed, 196 insertions(+), 42 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 241af2e0..85d4e586 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -639,10 +639,11 @@ jobs: echo "::error::Image revision/version labels do not match the selected checkout and requested features" exit 1 fi - python3 - <<'PY' + LABEL_VERSION="${VERSION}" LABEL_REVISION="${REVISION}" python3 - <<'PY' import hashlib import os import re + import subprocess from pathlib import Path def read_artifact(relative): path = Path(relative) @@ -653,12 +654,42 @@ jobs: except OSError: raise SystemExit('::error::Reviewed deployment artifact is unavailable') from None + def git(*args): + result = subprocess.run(['git', *args], capture_output=True, text=True) + return result.stdout if result.returncode == 0 else '' + path = Path('docker-compose.dstack-postgres.yaml') if path.is_symlink() or path.resolve() != Path.cwd().resolve() / path: raise SystemExit('::error::Deployment destination is not the fixed reviewed path') native = 'ghcr.io/tinycloudlabs/tinycloud-node' reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' - n0_hash = 'c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7' + # TC-959: #309's reviewed N0 compose (raw ca110b0b… with the 1.20.6 + # pin) with its single node digest normalized to 64 zeros. + n0_template_hash = '92be17ff125e5d4cada9b81d7a8e655442fa66a454e19f76e3f8880a93123d3d' + node_pin = re.compile(rb'(?m)^ image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$') + def n0_template(raw): + pins = list(node_pin.finditer(raw)) + if len(pins) != 1: + return False + normalized = raw[:pins[0].start(1)] + b'0' * 64 + raw[pins[0].end(1):] + return hashlib.sha256(normalized).hexdigest() == n0_template_hash + + def require_release(): + # Only a recorded, released main image may replace the N0 pin: + # its labels, vX.Y.Z tag and crate version must all agree. + refuse = '::error::Reviewed N0 compose accepts only a recorded released main image; deployment not attempted' + label = re.fullmatch(r'([0-9]+\.[0-9]+\.[0-9]+)-dstack', os.environ['LABEL_VERSION']) + revision = os.environ['LABEL_REVISION'] + if (not label or not re.fullmatch(r'[0-9a-f]{40}', revision) or not os.environ.get('INPUT_DEPLOY_DIGEST') + or os.environ['RESOLVED_IMAGE'] != native + '@' + os.environ['INPUT_DEPLOY_DIGEST'] + or os.environ.get('ROLLBACK_VERSION') != label[1] or os.environ.get('ROLLBACK_REVISION') != revision): + raise SystemExit(refuse) + if git('rev-parse', '--verify', '--quiet', f'refs/tags/v{label[1]}^{{commit}}').strip() != revision: + raise SystemExit(refuse) + crate = re.search(r'(?m)^version = "([^"]*)"$', git('show', f'{revision}:tinycloud-node-server/Cargo.toml')) + if not crate or crate[1] != label[1]: + raise SystemExit(refuse) + pre_n0_hash = 'df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91' pre_n0_template_hash = '28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93' rollback = os.environ.get('ROLLBACK_PRE_N0') == 'true' @@ -679,22 +710,28 @@ jobs: raise SystemExit('::error::Rollback privacy cannot modify a forward/current-compose deployment') raw = read_artifact(str(path)) digest = hashlib.sha256(raw).hexdigest() - n0_configuration = digest == n0_hash - if n0_configuration and os.environ['RESOLVED_IMAGE'] != reviewed: - raise SystemExit('::error::Reviewed N0 compose cannot substitute a different image') - if os.environ['RESOLVED_IMAGE'] == reviewed and digest not in (n0_hash, pre_n0_hash, pre_n0_template_hash): - raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose') + n0_configuration = n0_template(raw) + if os.environ['RESOLVED_IMAGE'] == reviewed: + if not n0_configuration and digest not in (pre_n0_hash, pre_n0_template_hash): + raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose') + elif n0_configuration: + require_release() content = raw.decode('utf-8') images = [ image for image in re.finditer(r'(?m)^([ \t]+image:[ \t]+)([^\n]+)$', content) if image[2].strip().strip('"\'').startswith((native + ':', native + '@')) ] - if len(images) != 1 or images[0][2].strip().strip('"\'') not in (native + ':dstack', reviewed): + value = images[0][2].strip().strip('"\'') if len(images) == 1 else '' + if not n0_configuration and value not in (native + ':dstack', reviewed): raise SystemExit('Expected exactly one :dstack or reviewed digest-pinned native image') - image = images[0] - if image[2].strip().strip('"\'') == reviewed and reviewed != os.environ['RESOLVED_IMAGE']: + if not n0_configuration and value == reviewed and reviewed != os.environ['RESOLVED_IMAGE']: raise SystemExit('Reviewed compose digest does not match the selected deployment image') - path.write_bytes((content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8')) + image = images[0] + deployed = (content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8') + selected_digest = os.environ['RESOLVED_IMAGE'].rsplit(':', 1)[1].encode() + if n0_configuration and (not n0_template(deployed) or [pin[1] for pin in node_pin.finditer(deployed)] != [selected_digest]): + raise SystemExit('::error::Deployed N0 compose must differ from the reviewed template only by the selected digest') + path.write_bytes(deployed) with open(os.environ['GITHUB_OUTPUT'], 'a') as output: output.write(f'n0_configuration_verified={str(n0_configuration).lower()}\n') output.write(f'rollback_pre_n0_verified={str(rollback).lower()}\n') diff --git a/docs/releasing.md b/docs/releasing.md index 396e90b5..42071897 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -66,12 +66,12 @@ main's history (`scripts/ci/check-deploy-ref.sh`, before builds and again inside `deploy-phala`). Other refs fail `Deploy refused (main-only)`. **Release order:** approved #310 must never merge alone. Land #310 and its -stacked recovery change atomically as one lineage, preserving emergency -re-promotion and security rollback first. #309's sidecar compose belongs -only to the later approved logging window, not this workflow/security -release. The security card remains rollback-not-ready until the combined -path is reviewed and landed. These docs grant no merge or dispatch authority. -Merging either workflow change is a push, not a deployment dispatch. +stacked recovery changes atomically as one lineage, preserving emergency +re-promotion and security rollback first. #309's sidecar compose may then +ship in the same window as a main release, promoted by one dispatch (see +[Main release with N0 log shipping](#main-release-with-n0-log-shipping-one-dispatch)). +These docs grant no merge or dispatch authority. Merging either workflow +change is a push, not a deployment dispatch. ### The `production` environment (TC-971) @@ -204,10 +204,13 @@ its branch policy before any workflow that names it runs. - `image_version` and recorded labels describe the immutable image, never an unrelated current crate version. The revision must be in main history. A future non-N0 digest uses this same contract. - - The reviewed N0 compose remains raw-hash locked to its exact image; - it cannot substitute a future security image. Before the later - logging window, the ordinary main `:dstack` compose supports forward - image promotions without installing the sidecar. + - The reviewed N0 compose is locked as a template: its raw bytes with + the single node digest normalized. Only a recorded *released* main + image may fill that pin; see + [Main release with N0 log shipping](#main-release-with-n0-log-shipping-one-dispatch). + Before #309 lands, + the ordinary main `:dstack` compose supports forward image + promotions without installing the sidecar. - `sql_identity_fence=false` is required. The input defaults to `true`, which re-fences SQL/DuckDB in production (503s). - The production ancestry guard still applies: the revision prod is running @@ -265,6 +268,54 @@ its branch policy before any workflow that names it runs. `gh workflow run write-probe.yml -R TinyCloudLabs/tinycloud-node --ref main -f mode=dense`. Details: `test/probe/README.md`. +### Main release with N0 log shipping (one dispatch) + +Ships a main release and #309's log sidecar together. Before dispatching, +`main` must contain the combined workflow lineage and #309, plus the merged +`chore(release): 1.21.0` commit tagged `v1.21.0`. Take the digest and +labels of the tag build's `1.21.0-dstack` image as in step 1 above: + +```sh +gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ + -f deploy_phala=true \ + -f authorize_quota_values=true \ + -f image_version=1.21.0 \ + -f deploy_image_digest=sha256: \ + -f rollback_image_version=1.21.0 \ + -f rollback_image_revision= \ + -f sql_identity_fence=false +``` + +Before any CVM mutation the workflow requires: + +- the main compose, with its one node digest replaced by 64 zeros, to hash + to the reviewed N0 template + **`92be17ff125e5d4cada9b81d7a8e655442fa66a454e19f76e3f8880a93123d3d`** + (#309's raw `ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6` + normalized). Ingress, socat, Vector and the log profile stay byte-exact; +- the deployed compose to differ from that template only by + `deploy_image_digest`; +- OCI labels `1.21.0-dstack` / ``, the `v1.21.0` tag on + that commit, its `Cargo.toml` at `1.21.0`, and the commit in main history. + An untagged, mislabelled, feature-variant or freshly built image is + refused. + +Sealed preflight, `BETTERSTACK_SOURCE_TOKEN` sealing with the +`allowed_envs` sync, and the general quota reconciliation run as on every +deploy. The template's comment naming 1.20.6 is reviewed text, not image +identity; `/version` and the labels report the release. Reproduce the +template hash from any ref: + +```sh +git show main:docker-compose.dstack-postgres.yaml \ + | sed -E '/^ image: ghcr\.io\/tinycloudlabs\/tinycloud-node@sha256:/s/[0-9a-f]{64}$/0000000000000000000000000000000000000000000000000000000000000000/' \ + | sha256sum +``` + +Rolling this release back means a database snapshot restore (a separate +decision) plus the [captured pre-N0 rollback](#tc-1302-captured-pre-n0-rollback) +to 1.20.6 with Sam's explicit privacy mode. + ## Rolling back Keep the digest, actual OCI version and actual revision of every promoted @@ -294,14 +345,14 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ ``` The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**. -The N0 raw lock **changed again in TC-1274**, which sets `TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"` in the production compose (whichever of this and #347 lands second re-pins). It previously changed in #346 (TC-959), which passes the validated N2 correlation fields through the log sidecar; the hash below is the TC-1274 lock: -`c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`, -from #346 final reviewed head. -It supersedes the #309 lock `ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6`, -from #309 reviewed head `44eeb20a1461ef66b5af492e3be4c42be2728b67`, -[approval](https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050). -Current pre-N0 tag/pinned equivalents are separately hash-checked; arbitrary -same-image config substitutions fail before mutation. +With #309 on main this redeploys exactly #309's reviewed raw compose +`ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6` +(from reviewed head `44eeb20a1461ef66b5af492e3be4c42be2728b67`, +[approval](https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050)), +whatever digest main's template currently pins. It is not a rollback of a +later release across a schema migration. Current pre-N0 tag/pinned +equivalents are separately hash-checked; arbitrary same-image config +substitutions fail before mutation. ### TC-1302 captured pre-N0 rollback diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py index 4df46cdf..246502ca 100755 --- a/scripts/ci/test-deploy-recovery.py +++ b/scripts/ci/test-deploy-recovery.py @@ -26,6 +26,12 @@ # reverted to the :dstack tag. The workflow accepts these bytes by raw hash. PRE_N0_TEMPLATE = PRE_N0.replace(b"@" + DIGEST.encode(), b":dstack") assert hashlib.sha256(PRE_N0_TEMPLATE).hexdigest() == "28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93" +RELEASE_DIGEST = "sha256:" + "b" * 64 +RELEASE_REVISION = "c" * 40 +RELEASE = {"deploy_image_digest": RELEASE_DIGEST, "image_version": "1.21.0", + "rollback_image_version": "1.21.0", "rollback_image_revision": RELEASE_REVISION} +RELEASED = {"1.21.0": (RELEASE_REVISION, "1.21.0")} +RELEASE_N0 = N0.replace(DIGEST.encode(), RELEASE_DIGEST.encode()) CANARIES = [] @@ -89,7 +95,19 @@ def event(value): assert args[-2:]==['config','--quiet']; assert Path('.phala-prod.env').stat().st_mode & 0o777==0o600 else: raise SystemExit('Unexpected Docker boundary') elif name=='git': - assert args==['rev-parse','HEAD']; print('f'*40) + releases=json.loads(os.environ['TEST_RELEASES']) + if args==['rev-parse','HEAD']: print('f'*40) + elif args[:3]==['rev-parse','--verify','--quiet'] and len(args)==4: + tag=args[3]; assert tag.startswith('refs/tags/v') and tag.endswith('^{commit}') + release=releases.get(tag[len('refs/tags/v'):-len('^{commit}')]) + if release is None: sys.exit(1) + print(release[0]) + elif args[0]=='show' and len(args)==2: + revision,path=args[1].split(':',1); assert path=='tinycloud-node-server/Cargo.toml' + crates=[crate for tagged,crate in releases.values() if tagged==revision] + if not crates: sys.exit(128) + print('[package]\nname = "tinycloud-node"\nversion = "'+crates[0]+'"') + else: raise SystemExit('Unexpected git boundary') elif name=='node': names=[line.split('=',1)[0] for line in Path('.phala-prod.env').read_text().splitlines()] assert 'BETTERSTACK_SOURCE_TOKEN' in names and 'QUOTA_BILLING_ADMIN_SECRET' not in names @@ -175,7 +193,7 @@ def _exc_info_to_string(self, error, test): class RecoveryTests(unittest.TestCase): def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_error="", authority="node-override", missing_billing=False, missing_authorization=False, approved_privacy="", event="workflow_dispatch", - ref="refs/heads/main", oci_label=None, oci_revision=None, build_digest=""): + ref="refs/heads/main", oci_label=None, oci_revision=None, build_digest="", releases=RELEASED): inputs = {"deploy_phala": "true", "authorize_quota_values": "true", "image_version": "1.20.6", "deploy_image_digest": DIGEST, "rollback_image_version": "1.20.6", "rollback_image_revision": REVISION, "rollback_pre_n0": "false", "rollback_privacy_mode": "unselected", "sql_identity_fence": "false"} @@ -231,7 +249,7 @@ def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_erro # Production maps both from PROD_TINYCLOUD_ADMIN_SECRET: one shared value. "QUOTA_ADMIN_SECRET": canary, "QUOTA_BILLING_ADMIN_SECRET": canary, # Production maps both from PROD_TINYCLOUD_QUOTA_URL: billing's base URL. - "QUOTA_BILLING_URL": BILLING_URL, "PHALA_ENV_TINYCLOUD_QUOTA_URL": BILLING_URL}) + "QUOTA_BILLING_URL": BILLING_URL, "PHALA_ENV_TINYCLOUD_QUOTA_URL": BILLING_URL, "TEST_RELEASES": json.dumps(releases)}) if missing_billing: env.pop("QUOTA_BILLING_ADMIN_SECRET") context = {"github.event_name": event, "github.ref": ref, "job.success": True, **{"inputs." + key: (value == "true" if value in ("true", "false") else value) for key, value in inputs.items()}} @@ -261,8 +279,8 @@ def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_erro "compose": compose_path.read_bytes(), "summary": report, "logs": logs} def test_recovery_current_pre_n0_and_reviewed_n0(self): - for compose in (PRE_N0_TEMPLATE, N0): - with self.subTest(n0=compose == N0): + for compose in (PRE_N0_TEMPLATE, PRE_N0, N0): + with self.subTest(n0=compose == N0, template=compose == PRE_N0_TEMPLATE): result = self.run_deploy(compose) self.assertTrue(result["success"], "Authorized same-image recovery must work") self.assertIn("sealed-preflight", result["events"]) @@ -278,6 +296,51 @@ def test_future_digest_gets_preflight_and_general_quota_restore(self): self.assertEqual(result["events"].count("quota-put"), 2) self.assertIn(b"@sha256:" + b"a" * 64, result["compose"]) + def test_release_digest_substitutes_only_the_reviewed_n0_node_digest(self): + # The reviewed template is N0 with its node digest normalized; the + # source pin is irrelevant, the deployed pin is exactly the release. + for source in (N0, RELEASE_N0): + with self.subTest(source_pinned_release=source == RELEASE_N0): + result = self.run_deploy(source, RELEASE) + self.assertTrue(result["success"], "Released digest must deploy the reviewed N0 template") + self.assertEqual(result["compose"], RELEASE_N0) + self.assertIn("sealed-preflight", result["events"]) + self.assertEqual(result["events"].count("quota-put"), 2) + self.assertIn("1.21.0-dstack", result["summary"]) + self.assertIn(RELEASE_REVISION, result["summary"]) + + def test_n0_release_requires_truthful_release_labels(self): + for name, releases in (("untagged", {}), ("tag elsewhere", {"1.21.0": ("d" * 40, "1.21.0")}), + ("crate disagrees", {"1.21.0": (RELEASE_REVISION, "1.20.7")})): + with self.subTest(name): + result = self.run_deploy(N0, RELEASE, releases=releases) + self.assertFalse(result["success"]) + self.assertNotIn("sealed-preflight", result["events"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) + with self.subTest("build path, not a recorded digest"): + result = self.run_deploy(N0, {"deploy_image_digest": "", "image_version": "1.20.7", "rollback_image_version": "", + "rollback_image_revision": ""}, build_digest=RELEASE_DIGEST, + releases={"1.20.7": ("f" * 40, "1.20.7")}) + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) + + def test_n0_template_drift_refuses_release_digest(self): + node_line = b" image: ghcr.io/tinycloudlabs/tinycloud-node@" + DIGEST.encode() + b"\n" + for name, compose in (("profile", N0.replace(b":-errors_slow}", b":-all_requests}")), + ("ingress", N0.replace(b"63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744", b"e" * 64)), + ("crlf", N0.replace(b"\n", b"\r\n")), + ("duplicate node", N0.replace(node_line, node_line * 2)), + ("quoted node", N0.replace(b"@" + DIGEST.encode(), b"@" + DIGEST.encode() + b"'").replace( + b"image: ghcr.io/tinycloudlabs/tinycloud-node@", b"image: 'ghcr.io/tinycloudlabs/tinycloud-node@"))): + with self.subTest(name): + self.assertNotEqual(compose, N0) + result = self.run_deploy(compose, RELEASE) + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], compose) + def test_explicit_rollback_privacy_variants(self): for mode, expected in (("keep-json-file", PRE_N0), ("stop-future-local-logs", STOP_LOGS)): with self.subTest(mode=mode): @@ -348,11 +411,11 @@ def test_normal_build_deployment_keeps_preflight_and_general_quota(self): self.assertIn("sealed-preflight", result["events"]) self.assertEqual(result["events"].count("quota-put"), 2) - def test_forward_n0_lock_and_captured_rollback_are_not_substitutable(self): + def test_unreleased_digest_and_captured_rollback_are_not_substitutable(self): future = {"deploy_image_digest": "sha256:" + "a" * 64, "image_version": "1.20.8", "rollback_image_version": "1.20.8", "rollback_image_revision": "a" * 40} - for changes, mode in ((future, ""), ({**future, "rollback_pre_n0": "true", - "rollback_privacy_mode": "keep-json-file"}, "keep-json-file")): + rollback = {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"} + for changes, mode in ((future, ""), ({**future, **rollback}, "keep-json-file"), ({**RELEASE, **rollback}, "keep-json-file")): result = self.run_deploy(N0, changes, approved_privacy=mode) self.assertFalse(result["success"]) self.assertNotIn("cvm-deploy", result["events"]) @@ -419,23 +482,26 @@ def test_three_shell_blocks_keep_strict_runtime_options(self): capture_output=True, text=True) self.assertEqual(trace.read_text().splitlines(), ["strict"]) - def test_documented_recovery_and_both_rollback_commands(self): + def test_documented_recovery_rollback_and_release_commands(self): text = (ROOT / "docs/releasing.md").read_text() - for heading in ("### Emergency same-image recovery (12c)", "### TC-1302 captured pre-N0 rollback"): + for heading, count, compose in (("### Emergency same-image recovery (12c)", 1, CURRENT), + ("### TC-1302 captured pre-N0 rollback", 2, CURRENT), + ("### Main release with N0 log shipping (one dispatch)", 1, N0)): self.assertTrue(heading in text, "Copy-paste recovery contract is missing") section = text.split(heading, 1)[1].split("\n### ", 1)[0].split("\n## ", 1)[0] commands = re.findall(r"```sh\n(gh workflow run docker.yml.*?)\n```", section, re.S) - self.assertEqual(len(commands), 1 if "12c" in heading else 2) + self.assertEqual(len(commands), count) for command in commands: + command = command.replace("", "b" * 64).replace("", RELEASE_REVISION) tokens = shlex.split(command.replace("\\\n", " ")) self.assertEqual(tokens[:6], ["gh", "workflow", "run", "docker.yml", "-R", "TinyCloudLabs/tinycloud-node"]) self.assertEqual(tokens[tokens.index("--ref") + 1], "main") inputs = dict(tokens[index + 1].split("=", 1) for index, value in enumerate(tokens) if value == "-f") mode = inputs.get("rollback_privacy_mode", "") - result = self.run_deploy(changes=inputs, approved_privacy=mode) + result = self.run_deploy(compose, inputs, approved_privacy=mode) self.assertTrue(result["success"], "Documented command failed actual input validation") inputs.pop("authorize_quota_values") - denied = self.run_deploy(changes=inputs, missing_authorization=True, approved_privacy=mode) + denied = self.run_deploy(compose, inputs, missing_authorization=True, approved_privacy=mode) self.assertFalse(denied["success"]) self.assertEqual(denied["events"], []) From f70f2b95f7ee1fb4bb0f71ff002697a5095d28fb Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 03:55:18 +0000 Subject: [PATCH 2/3] ci: carry the 95m ingress body cap in the N0 release template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise CLIENT_MAX_BODY_SIZE 32m -> 95m in the root compose (#309's approved ca110b0b… becomes b0cd8248…, 2 bytes) and lock the release template over those bytes: 92641ed5…. #309's approved pre-cap bytes no longer satisfy the template. Pre-N0 template and captured rollback artifacts are unchanged. Refs TC-959, TC-1012 --- .github/workflows/docker.yml | 6 ++--- docker-compose.dstack-postgres.yaml | 2 +- docs/releasing.md | 31 +++++++++++++---------- docs/runbooks/tc-959-n0-log-shipping.md | 1 + scripts/ci/fixtures/tc959-n0-compose.yaml | 2 +- scripts/ci/test-deploy-recovery.py | 1 + 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 85d4e586..2b7e366e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -663,9 +663,9 @@ jobs: raise SystemExit('::error::Deployment destination is not the fixed reviewed path') native = 'ghcr.io/tinycloudlabs/tinycloud-node' reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' - # TC-959: #309's reviewed N0 compose (raw ca110b0b… with the 1.20.6 - # pin) with its single node digest normalized to 64 zeros. - n0_template_hash = '92be17ff125e5d4cada9b81d7a8e655442fa66a454e19f76e3f8880a93123d3d' + # TC-959: main's N0 compose (#349 c33c2302…) with the 95m ingress body + # cap (raw d95bfb70…), its single node digest normalized to 64 zeros. + n0_template_hash = '51f8e9e93edc0e16025d27ee42abe003ed5d1cf26ba8f437d5120894aca64bdb' node_pin = re.compile(rb'(?m)^ image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$') def n0_template(raw): pins = list(node_pin.finditer(raw)) diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index 24da7bc4..535642ab 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -111,7 +111,7 @@ services: - CERTBOT_EMAIL=${CERTBOT_EMAIL} - SET_CAA=true - DNS_PROVIDER=cloudflare - - CLIENT_MAX_BODY_SIZE=32m + - CLIENT_MAX_BODY_SIZE=95m volumes: - /var/run/dstack.sock:/var/run/dstack.sock - cert-data:/etc/letsencrypt diff --git a/docs/releasing.md b/docs/releasing.md index 42071897..eea847c6 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -67,8 +67,10 @@ inside `deploy-phala`). Other refs fail `Deploy refused (main-only)`. **Release order:** approved #310 must never merge alone. Land #310 and its stacked recovery changes atomically as one lineage, preserving emergency -re-promotion and security rollback first. #309's sidecar compose may then -ship in the same window as a main release, promoted by one dispatch (see +re-promotion and security rollback first. #309's sidecar compose then ships +on the unchanged 1.20.6 image under that lineage's exact raw-hash lock. The +later main release lands this template lock and the 95m ingress body cap, +and is promoted by one dispatch (see [Main release with N0 log shipping](#main-release-with-n0-log-shipping-one-dispatch)). These docs grant no merge or dispatch authority. Merging either workflow change is a push, not a deployment dispatch. @@ -270,9 +272,10 @@ its branch policy before any workflow that names it runs. ### Main release with N0 log shipping (one dispatch) -Ships a main release and #309's log sidecar together. Before dispatching, -`main` must contain the combined workflow lineage and #309, plus the merged -`chore(release): 1.21.0` commit tagged `v1.21.0`. Take the digest and +Ships a main release on #309's log sidecar compose. Before dispatching, +`main` must contain the combined workflow lineage, #309, the template-lock +change that also raises the ingress `CLIENT_MAX_BODY_SIZE` to `95m`, and the +merged `chore(release): 1.21.0` commit tagged `v1.21.0`. Take the digest and labels of the tag build's `1.21.0-dstack` image as in step 1 above: ```sh @@ -290,9 +293,12 @@ Before any CVM mutation the workflow requires: - the main compose, with its one node digest replaced by 64 zeros, to hash to the reviewed N0 template - **`92be17ff125e5d4cada9b81d7a8e655442fa66a454e19f76e3f8880a93123d3d`** - (#309's raw `ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6` - normalized). Ingress, socat, Vector and the log profile stay byte-exact; + **`51f8e9e93edc0e16025d27ee42abe003ed5d1cf26ba8f437d5120894aca64bdb`** + (raw `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50` normalized: the #349 compose + `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`, which + keeps `TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"` and #346's Vector + config, with only `CLIENT_MAX_BODY_SIZE=32m` → `95m`). Ingress, socat, + Vector and the log profile stay byte-exact; - the deployed compose to differ from that template only by `deploy_image_digest`; - OCI labels `1.21.0-dstack` / ``, the `v1.21.0` tag on @@ -345,11 +351,10 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ ``` The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**. -With #309 on main this redeploys exactly #309's reviewed raw compose -`ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6` -(from reviewed head `44eeb20a1461ef66b5af492e3be4c42be2728b67`, -[approval](https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050)), -whatever digest main's template currently pins. It is not a rollback of a +With the template lock on main this redeploys exactly main's raw compose +`d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50` +(the #349 compose `c33c2302…` plus the 95m ingress body cap), whatever digest +main's template currently pins. It is not a rollback of a later release across a schema migration. Current pre-N0 tag/pinned equivalents are separately hash-checked; arbitrary same-image config substitutions fail before mutation. diff --git a/docs/runbooks/tc-959-n0-log-shipping.md b/docs/runbooks/tc-959-n0-log-shipping.md index 39a944c4..8a23367a 100644 --- a/docs/runbooks/tc-959-n0-log-shipping.md +++ b/docs/runbooks/tc-959-n0-log-shipping.md @@ -147,6 +147,7 @@ Keep these identities distinct: | Before: verifier `docker_compose_file` / hash | Raw preserved pre-N0 compose: `df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91` | | Before: application `compose_hash` | `bfe9028a58b23238188aa7d70b82282a7cd6414cd83dcc47a3b295c6f64fe2f8`; a distinct reported identity, not the raw-file hash | | After: current profile-enabled raw compose | `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`; bind this artifact to the final reviewed PR head before any promotion | +| After the main release: template-locked raw compose | `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50` (main's compose with only `CLIENT_MAX_BODY_SIZE=32m` → `95m`); node digest normalized, template `51f8e9e93edc0e16025d27ee42abe003ed5d1cf26ba8f437d5120894aca64bdb`. See [`docs/releasing.md`](../releasing.md#main-release-with-n0-log-shipping-one-dispatch) | The raw after hash identifies the profile-enabled compose, not deployment approval. Record its final reviewed PR head and required CI evidence in the officer handoff. Do not infer a raw-file hash from the distinct application-reported compose hash. diff --git a/scripts/ci/fixtures/tc959-n0-compose.yaml b/scripts/ci/fixtures/tc959-n0-compose.yaml index 24da7bc4..535642ab 100644 --- a/scripts/ci/fixtures/tc959-n0-compose.yaml +++ b/scripts/ci/fixtures/tc959-n0-compose.yaml @@ -111,7 +111,7 @@ services: - CERTBOT_EMAIL=${CERTBOT_EMAIL} - SET_CAA=true - DNS_PROVIDER=cloudflare - - CLIENT_MAX_BODY_SIZE=32m + - CLIENT_MAX_BODY_SIZE=95m volumes: - /var/run/dstack.sock:/var/run/dstack.sock - cert-data:/etc/letsencrypt diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py index 246502ca..c2428a4c 100755 --- a/scripts/ci/test-deploy-recovery.py +++ b/scripts/ci/test-deploy-recovery.py @@ -330,6 +330,7 @@ def test_n0_template_drift_refuses_release_digest(self): node_line = b" image: ghcr.io/tinycloudlabs/tinycloud-node@" + DIGEST.encode() + b"\n" for name, compose in (("profile", N0.replace(b":-errors_slow}", b":-all_requests}")), ("ingress", N0.replace(b"63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744", b"e" * 64)), + ("approved pre-cap bytes", N0.replace(b"CLIENT_MAX_BODY_SIZE=95m", b"CLIENT_MAX_BODY_SIZE=32m")), ("crlf", N0.replace(b"\n", b"\r\n")), ("duplicate node", N0.replace(node_line, node_line * 2)), ("quoted node", N0.replace(b"@" + DIGEST.encode(), b"@" + DIGEST.encode() + b"'").replace( From 7ad6aa42d861e3e7147cf97aa9a85302c000d808 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 10:51:42 +0000 Subject: [PATCH 3/3] ci: refuse release downgrades under the N0 template and old same-image recovery The promoted revision must strictly descend from the running production revision; 12c same-image recovery refuses an image older than running. The captured pre-N0 rollback stays the intentional downgrade route. Refs TC-959 --- .github/workflows/docker.yml | 19 +++++++++++++ docs/releasing.md | 13 ++++++++- scripts/ci/test-deploy-recovery.py | 44 ++++++++++++++++++++++++++++-- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2b7e366e..970bc012 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -594,6 +594,8 @@ jobs: exit 1 fi echo "Running production revision ${RUNNING_REVISION} is an ancestor of ${GITHUB_SHA}." + # Later steps refuse a deployment older than what is running. + echo "RUNNING_REVISION=${RUNNING_REVISION}" >> "$GITHUB_ENV" - name: Preflight quota authority env: @@ -689,6 +691,22 @@ jobs: crate = re.search(r'(?m)^version = "([^"]*)"$', git('show', f'{revision}:tinycloud-node-server/Cargo.toml')) if not crate or crate[1] != label[1]: raise SystemExit(refuse) + require_release_descends(revision) + + def is_ancestor(older, newer): + return subprocess.run(['git', 'merge-base', '--is-ancestor', older, newer], capture_output=True).returncode == 0 + + running = os.environ.get('RUNNING_REVISION', '') + + def require_not_older_than_running(revision): + # No cross-migration downgrade: the running production revision + # must be the deployed image's revision or its ancestor. + if running and not is_ancestor(running, revision): + raise SystemExit('::error::Refusing to deploy an image older than the running production revision; use the captured pre-N0 rollback for an intentional rollback') + + def require_release_descends(revision): + if not re.fullmatch(r'[0-9a-f]{40}', running) or running == revision or not is_ancestor(running, revision): + raise SystemExit('::error::Released image must be a strict descendant of the running production revision; deployment not attempted') pre_n0_hash = 'df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91' pre_n0_template_hash = '28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93' @@ -714,6 +732,7 @@ jobs: if os.environ['RESOLVED_IMAGE'] == reviewed: if not n0_configuration and digest not in (pre_n0_hash, pre_n0_template_hash): raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose') + require_not_older_than_running(os.environ['LABEL_REVISION']) elif n0_configuration: require_release() content = raw.decode('utf-8') diff --git a/docs/releasing.md b/docs/releasing.md index eea847c6..4d2c0f6f 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -289,6 +289,10 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ -f sql_identity_fence=false ``` +`rollback_image_version` and `rollback_image_revision` name the image being +**deployed** (here the release), not an image to roll back to. Despite the +input names, they are the recorded labels of `deploy_image_digest`. + Before any CVM mutation the workflow requires: - the main compose, with its one node digest replaced by 64 zeros, to hash @@ -304,7 +308,14 @@ Before any CVM mutation the workflow requires: - OCI labels `1.21.0-dstack` / ``, the `v1.21.0` tag on that commit, its `Cargo.toml` at `1.21.0`, and the commit in main history. An untagged, mislabelled, feature-variant or freshly built image is - refused. + refused; +- the release revision to be a **strict descendant** of the revision running + in production (read from the CVM's pinned image by the ancestry guard). + A downgrade (for example 1.20.5 or 1.0.0), the same revision as running, + or an unknown running revision is refused. The same ancestry rule makes + the 12c same-image command refuse an image older than the running + revision once 1.21.0 is live, so it cannot downgrade across migrations. + The captured pre-N0 rollback is the only intentional downgrade route. Sealed preflight, `BETTERSTACK_SOURCE_TOKEN` sealing with the `allowed_envs` sync, and the general quota reconciliation run as on every diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py index c2428a4c..7d49fef3 100755 --- a/scripts/ci/test-deploy-recovery.py +++ b/scripts/ci/test-deploy-recovery.py @@ -32,6 +32,9 @@ "rollback_image_version": "1.21.0", "rollback_image_revision": RELEASE_REVISION} RELEASED = {"1.21.0": (RELEASE_REVISION, "1.21.0")} RELEASE_N0 = N0.replace(DIGEST.encode(), RELEASE_DIGEST.encode()) +OLD_REVISION = "a" * 40 +ANCIENT_REVISION = "9" * 40 +ORDER = [ANCIENT_REVISION, OLD_REVISION, REVISION, RELEASE_REVISION] CANARIES = [] @@ -97,6 +100,10 @@ def event(value): elif name=='git': releases=json.loads(os.environ['TEST_RELEASES']) if args==['rev-parse','HEAD']: print('f'*40) + elif args[:2]==['merge-base','--is-ancestor'] and len(args)==4: + order=json.loads(os.environ['TEST_ORDER']) + ok=args[2] in order and args[3] in order and order.index(args[2])<=order.index(args[3]) + sys.exit(0 if ok else 1) elif args[:3]==['rev-parse','--verify','--quiet'] and len(args)==4: tag=args[3]; assert tag.startswith('refs/tags/v') and tag.endswith('^{commit}') release=releases.get(tag[len('refs/tags/v'):-len('^{commit}')]) @@ -193,7 +200,7 @@ def _exc_info_to_string(self, error, test): class RecoveryTests(unittest.TestCase): def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_error="", authority="node-override", missing_billing=False, missing_authorization=False, approved_privacy="", event="workflow_dispatch", - ref="refs/heads/main", oci_label=None, oci_revision=None, build_digest="", releases=RELEASED): + ref="refs/heads/main", oci_label=None, oci_revision=None, build_digest="", releases=RELEASED, running=REVISION, order=None): inputs = {"deploy_phala": "true", "authorize_quota_values": "true", "image_version": "1.20.6", "deploy_image_digest": DIGEST, "rollback_image_version": "1.20.6", "rollback_image_revision": REVISION, "rollback_pre_n0": "false", "rollback_privacy_mode": "unselected", "sql_identity_fence": "false"} @@ -249,7 +256,9 @@ def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_erro # Production maps both from PROD_TINYCLOUD_ADMIN_SECRET: one shared value. "QUOTA_ADMIN_SECRET": canary, "QUOTA_BILLING_ADMIN_SECRET": canary, # Production maps both from PROD_TINYCLOUD_QUOTA_URL: billing's base URL. - "QUOTA_BILLING_URL": BILLING_URL, "PHALA_ENV_TINYCLOUD_QUOTA_URL": BILLING_URL, "TEST_RELEASES": json.dumps(releases)}) + "QUOTA_BILLING_URL": BILLING_URL, "PHALA_ENV_TINYCLOUD_QUOTA_URL": BILLING_URL, "TEST_RELEASES": json.dumps(releases), + "TEST_ORDER": json.dumps(order or ORDER)}) + if running: env["RUNNING_REVISION"] = running if missing_billing: env.pop("QUOTA_BILLING_ADMIN_SECRET") context = {"github.event_name": event, "github.ref": ref, "job.success": True, **{"inputs." + key: (value == "true" if value in ("true", "false") else value) for key, value in inputs.items()}} @@ -326,6 +335,37 @@ def test_n0_release_requires_truthful_release_labels(self): self.assertNotIn("cvm-deploy", result["events"]) self.assertEqual(result["compose"], N0) + def test_release_must_descend_from_the_running_revision(self): + digest = lambda ch: "sha256:" + ch * 64 + def release(version, revision, ch): + return ({"deploy_image_digest": digest(ch), "image_version": version, "rollback_image_version": version, + "rollback_image_revision": revision}, {version: (revision, version)}) + for name, version, revision, ch, running, ok in ( + ("1.21.0 over running 1.20.6", "1.21.0", RELEASE_REVISION, "b", REVISION, True), + ("1.20.5 downgrade", "1.20.5", OLD_REVISION, "d", REVISION, False), + ("1.0.0 downgrade", "1.0.0", ANCIENT_REVISION, "e", REVISION, False), + ("same revision as running", "1.21.0", RELEASE_REVISION, "b", RELEASE_REVISION, False), + ("running revision unknown", "1.21.0", RELEASE_REVISION, "b", "", False)): + with self.subTest(name): + changes, releases = release(version, revision, ch) + result = self.run_deploy(N0, changes, releases=releases, running=running) + self.assertEqual(result["success"], ok) + if not ok: + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) + + def test_same_image_recovery_refuses_an_older_image_than_running(self): + for compose in (PRE_N0_TEMPLATE, N0): + with self.subTest(n0=compose == N0): + result = self.run_deploy(compose, running=RELEASE_REVISION) + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], compose) + with self.subTest("explicit rollback is exempt"): + result = self.run_deploy(N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, + approved_privacy="keep-json-file", running=RELEASE_REVISION) + self.assertTrue(result["success"]) + def test_n0_template_drift_refuses_release_digest(self): node_line = b" image: ghcr.io/tinycloudlabs/tinycloud-node@" + DIGEST.encode() + b"\n" for name, compose in (("profile", N0.replace(b":-errors_slow}", b":-all_requests}")),