TC-1482: wire optional NODE_STATS_READ_TOKEN into the production deploy (companion to #370)

.github/workflows/docker.yml

@@ -666,9 +666,10 @@ jobs:
           native = 'ghcr.io/tinycloudlabs/tinycloud-node'
           reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f'
           # TC-959: main's N0 compose (#349 c33c2302…) with the 95m ingress body
-          # cap (raw d95bfb70…) and the TC-1274 + TC-1317 Vector config
-          # (raw 1228d819…), its single node digest normalized to 64 zeros.
-          n0_template_hash = '7fabba70e71423faa2c8b61981d4a7c95ac7c2552074af81c0877af774d6b383'
+          # cap (raw d95bfb70…), the TC-1274 + TC-1317 + TC-1482 + c_index Vector config and the
+          # optional TC-1482 NODE_STATS_READ_TOKEN env line (raw 32a002ef…),
+          # its single node digest normalized to 64 zeros.
+          n0_template_hash = '79469f97a8dcecc7856caa11b26e3004b36d4b8ae9e16a51f963d0687992b9fb'
           node_pin = re.compile(rb'(?m)^    image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$')
           def n0_template(raw):
               pins = list(node_pin.finditer(raw))
@@ -926,6 +927,7 @@ jobs:
           PHALA_ENV_AWS_REGION: ${{ secrets.PROD_TINYCLOUD_AWS_REGION }}
           PHALA_ENV_TINYCLOUD_QUOTA_URL: ${{ secrets.PROD_TINYCLOUD_QUOTA_URL }}
           PHALA_ENV_TINYCLOUD_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }}
+          PHALA_ENV_NODE_STATS_READ_TOKEN: ${{ secrets.PROD_TINYCLOUD_NODE_STATS_READ_TOKEN }}
           PHALA_ENV_SHARE_TRUST_BUNDLE_BASE64: ${{ secrets.PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64 }}
           PHALA_ENV_CLOUDFLARE_API_TOKEN: ${{ secrets.PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN }}
           PHALA_ENV_DSTACK_GATEWAY_DOMAIN: ${{ secrets.PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN }}
@@ -970,6 +972,31 @@ jobs:
             fi
           done
 
+          # Optional (TC-1482): the same single-line rule, but absence is allowed.
+          if [[ "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" == *$'\n'* || "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" == *$'\r'* ]]; then
+            echo "::error::Phala deploy inputs must be single-line values"
+            exit 1
+          fi
+
+          # TC-1482: the node keeps GET /admin/stats disabled (404) when the token
+          # equals TINYCLOUD_ADMIN_SECRET or is shorter than 32 bytes (surrounding
+          # whitespace ignored), which would surface only as an OpenKey 503. Refuse
+          # both here, before anything is written, with constant messages; no value
+          # is printed. `wc -c` counts bytes.
+          if [ -n "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" ]; then
+            if [ "$PHALA_ENV_NODE_STATS_READ_TOKEN" = "$PHALA_ENV_TINYCLOUD_ADMIN_SECRET" ]; then
+              echo "::error::NODE_STATS_READ_TOKEN must differ from TINYCLOUD_ADMIN_SECRET"
+              exit 1
+            fi
+            stats_token_trimmed="${PHALA_ENV_NODE_STATS_READ_TOKEN#"${PHALA_ENV_NODE_STATS_READ_TOKEN%%[![:space:]]*}"}"
+            stats_token_trimmed="${stats_token_trimmed%"${stats_token_trimmed##*[![:space:]]}"}"
+            if [ "$(printf '%s' "$stats_token_trimmed" | wc -c)" -lt 32 ]; then
+              echo "::error::NODE_STATS_READ_TOKEN must be at least 32 bytes"
+              exit 1
+            fi
+            unset stats_token_trimmed
+          fi
+
           # Also restrict a file left by an earlier failed runner step before
           # writing any value; umask alone does not tighten an existing file.
           : > .phala-prod.env
@@ -989,6 +1016,11 @@ jobs:
             printf 'CERTBOT_EMAIL=%s\n' "$PHALA_ENV_CERTBOT_EMAIL"
             printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN"
             printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE"
+            # TC-1482: optional. Unset keeps the node starting with
+            # GET /admin/stats disabled (404); never written empty.
+            if [ -n "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" ]; then
+              printf 'NODE_STATS_READ_TOKEN=%s\n' "$PHALA_ENV_NODE_STATS_READ_TOKEN"
+            fi
             # SSH public keys authorized on the CVM (checked into the repo,
             # see dstack-authorized-keys). The Phala Cloud pre-launch script
             # writes DSTACK_AUTHORIZED_KEYS to /home/root/.ssh/authorized_keys

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## [Unreleased]
 
+### Changes
+
+- `/admin/stats` is a logged route label (TC-1482). The log shipper and the ingress log it as the route template plus status only, with no query string, `Authorization` header, request or response body, or client address (TC-961).
+  - **N0 compose.** The embedded Vector configuration is regenerated for the new route. Together with the optional `NODE_STATS_READ_TOKEN` env line, the reviewed N0 compose template changes to `79469f97…` (raw `32a002ef…`; `docs/releasing.md`).
+  - **`current_kv` C-index warning codes.** The log sidecar maps the three constant warnings of the TC-1274 `current_kv` C-index migration to `current_kv_c_index_missing`, `current_kv_c_index_invalid` and `current_kv_c_index_wrong_shape` (TC-1274); until then they would ship as `unclassified`. This is part of the same regenerated Vector configuration.
+  - **Ingress.** The `/admin/stats` label takes effect once the ingress image is rebuilt from `deploy/ingress` and its digest is pinned in the compose; until then the deployed ingress logs it as `<other>`.
+
 ## [1.21.2] - 2026-10-19
 
 ### Release notes

deploy/ingress/redacted-logging.conf

@@ -24,6 +24,7 @@ map $uri $tc_route {
     /signed/kv                                      /signed/kv;
     /admin/quota                                    /admin/quota;
     /admin/usage                                    /admin/usage;
+    /admin/stats                                    /admin/stats;
     /hooks/events                                   /hooks/events;
     /hooks/tickets                                  /hooks/tickets;
     /hooks/webhooks                                 /hooks/webhooks;

deploy/ingress/test-redacted-logging.sh

@@ -43,6 +43,7 @@ echo "testing $IMAGE"
 hex() { od -An -N"$1" -tx1 /dev/urandom | tr -d ' \n'; }
 TICKET="tc970ticket$(hex 12)"
 HOOK_TICKET="tc970hookticket$(hex 12)"
+STATS_TOKEN="tc1482statstoken$(hex 12)"
 WALLET="$(hex 20)"
 SPACE="tinycloud:pkh:eip155:1:0x${WALLET}:default"
 KEY="tc970key$(hex 12)"
@@ -128,6 +129,8 @@ expect_status() {
 
 expect_status 401 "https://$DOMAIN/signed/kv/$TICKET"
 expect_status 200 "https://$DOMAIN/hooks/events?ticket=$HOOK_TICKET"
+# TC-1482: /admin/stats is its own route; its query string never reaches the log.
+expect_status 200 "https://$DOMAIN/admin/stats?token=$STATS_TOKEN"
 expect_status 502 "https://$DOMAIN/public/$SPACE/kv/$KEY"
 expect_status 401 "https://$DOMAIN/signed/kv/$TICKET/unmatched"
 # HTTP/1.1, a client-chosen method, and a 400 raised before routing (a header
@@ -156,6 +159,7 @@ FILES=$($DOCKER exec "$INGRESS" sh -c 'find /var/log/nginx -type f -exec cat {}
 for pair in \
     "signed-KV ticket=$TICKET" \
     "hook ticket=$HOOK_TICKET" \
+    "admin stats query token=$STATS_TOKEN" \
     "space id=$SPACE" \
     "wallet address=$WALLET" \
     "KV key=$KEY" \
@@ -200,6 +204,7 @@ expect_line() {
 }
 expect_line GET "/signed/kv/<ticket_id>" 401
 expect_line GET "/hooks/events" 200
+expect_line GET "/admin/stats" 200
 expect_line GET "/public/<space_id>/kv/<key..>" 502
 expect_line GET "<other>" 401
 expect_line "<other>" "/signed/kv/<ticket_id>" 401

docs/releasing.md

@@ -147,6 +147,67 @@ a bare HTTPS origin. The preflight also accepts that origin with the
 or CR/LF fails it. The quota steps take only its host and port and
 call billing's fixed `/api/admin/storage-comp/` path; no URL is printed.
 
+### `NODE_STATS_READ_TOKEN` (TC-1482): compose hash change
+
+`docker-compose.dstack-postgres.yaml` passes the optional bearer for the
+read-only `GET /admin/stats` to the node as `NODE_STATS_READ_TOKEN:
+"${NODE_STATS_READ_TOKEN:-}"`. This ships in the same release as the route
+(the release after 1.21.2) and **changes the compose hash**: the N0 template
+lock in `docker.yml` moves to the hash above and
+`scripts/ci/fixtures/tc959-n0-compose.yaml` stays byte-identical to the
+compose. Release notes for that release must say so.
+
+- The secret is the `production` environment secret
+  `PROD_TINYCLOUD_NODE_STATS_READ_TOKEN`. The deploy step maps it to
+  `PHALA_ENV_NODE_STATS_READ_TOKEN` and writes it to the sealed
+  `.phala-prod.env` (mode 0600, removed afterwards) like the other secrets;
+  the value is never in a file in the repository, in argv or in logs.
+- It is **optional**. It is not in the deploy step's `required_vars`. Unset
+  or empty, the line is not written, the node starts normally and the route
+  is disabled (`404`, as for an unknown route). `--validate-config` and the
+  compose validation do not read it. A multi-line value fails before any CVM
+  mutation.
+- When set, the deploy step **refuses**, before it writes the sealed file and
+  with a constant message that names no value, a token that equals
+  `PROD_TINYCLOUD_ADMIN_SECRET` (`NODE_STATS_READ_TOKEN must differ from
+  TINYCLOUD_ADMIN_SECRET`) and a token shorter than 32 bytes
+  (`NODE_STATS_READ_TOKEN must be at least 32 bytes`; surrounding whitespace
+  is not counted). The node itself keeps the route at `404` for either value,
+  so without this check the mistake would surface only as OpenKey's `503`.
+- `ops/rollback/pre-n0-compose.yaml` and `pre-n0-stop-local-logs.yaml` are not
+  changed: they are pinned by raw hash, restore the 1.20.6 image, which does
+  not read this variable, and the rollback leaves the route absent.
+
+Items for the release notes of the release that ships this:
+
+- **(a) Registration.** No registration is needed if production uses Phala's
+  centralized KMS. Amos confirms that before the deploy with
+  `phala cvms get tinycloud-node` (output stays private). If it is an on-chain
+  KMS, a reviewed `addComposeHash` is needed first. dstack's `compose_hash`
+  changes at the deploy, and again when the first deploy adds the token name
+  to `allowed_envs`.
+- **(b) Node DID.** The node DID is unchanged across the compose-hash change:
+  the key path is `tinycloud/keys/primary`, derived from the app id and that
+  path. The post-deploy DID and key-fingerprint gate stays.
+- **(c) Rule: never delete or blank `PROD_TINYCLOUD_NODE_STATS_READ_TOKEN`
+  while the name is in the CVM's `allowed_envs`.** A later deploy without it
+  fails closed before any CVM mutation with exactly
+  `::error::allowed_envs synchronization failed during complete whitelist validation; response details withheld.`
+  (the whitelist check requires every allowed name to be present in the sealed
+  file), and it also blocks rollback and same-image recovery, which use the same
+  deploy step. Removing the name later is a manual, reviewed Phala
+  control-plane operation: set `allowed_envs` without the name and a
+  re-encrypted env set without it (this changes `compose_hash` again), then
+  delete the secret.
+- **(d) First deploy.** The first deploy with the token set restarts the CVM
+  twice: dstack skips the not-yet-allowed key, so the route is `404` on that
+  boot, and the post-deploy sync adds the name. [inference from the sync code;
+  not observed live]
+- **(e) Token.** Bobbie generates a 32-byte hex token (`openssl rand -hex 32`,
+  64 characters). Amos puts it into the Phala env through the deploy, by
+  setting the `production` environment secret from stdin, never printed. The
+  same value is set as OpenKey's `NODE_STATS_READ_TOKEN`.
+
 Every `deploy_phala=true` command must explicitly include
 **`-f authorize_quota_values=true`**. This authorizes the reviewed values
 and authority in `ops/quota-overrides.json` for that one dispatch; it is
@@ -309,11 +370,11 @@ Before any CVM mutation the workflow requires:
 
 - the main compose, with its one node digest replaced by 64 zeros, to hash
   to the reviewed N0 template
-  **`7fabba70e71423faa2c8b61981d4a7c95ac7c2552074af81c0877af774d6b383`**
-  (raw `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350` normalized: the 1.21.0 compose
+  **`79469f97a8dcecc7856caa11b26e3004b36d4b8ae9e16a51f963d0687992b9fb`**
+  (raw `32a002ef5751817619423c9e60776e03f2c3517315f5aa3a9c8654e3394777f1` normalized; before the three `current_kv_c_index_*` warning codes in the Vector config (TC-1274 `current_kv` C-index migration) the template was `87b5d12a…`, raw `d1fcd665…`; before the optional TC-1482 `NODE_STATS_READ_TOKEN` env line and the TC-1482 route in the Vector config the template was `7fabba70…`, raw `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350`: the 1.21.0 compose
   `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50`, template
-  `51f8e9e9…`, with only the embedded Vector config regenerated for TC-1274's
-  and TC-1317's warning codes; TC-1274 alone was template `00938226…`, raw
+  `51f8e9e9…`, with only the embedded Vector config regenerated for TC-1274's,
+  TC-1317's and TC-1482's route and warning codes (TC-1274 and TC-1317 were template `7fabba70…`, raw `1228d819…`); TC-1274 alone was template `00938226…`, raw
   `b25b8414…`. That compose is the #349 compose
   `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`, which
   keeps `TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"`, with only
@@ -379,8 +440,8 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \
 
 The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**.
 With the template lock on main this redeploys exactly main's raw compose
-`1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350`
-(the 1.21.0 compose `d95bfb70…` with the TC-1274 and TC-1317 Vector config), whatever digest
+`32a002ef5751817619423c9e60776e03f2c3517315f5aa3a9c8654e3394777f1`
+(the 1.21.0 compose `d95bfb70…` with the TC-1274, TC-1317, TC-1482 and `current_kv_c_index_*` Vector config and the optional TC-1482 `NODE_STATS_READ_TOKEN` env line), whatever digest
 main's template currently pins. It is not a rollback of a
 later release across a schema migration. Current pre-N0 tag/pinned
 equivalents are separately hash-checked; arbitrary same-image config

docs/runbooks/tc-959-n0-log-shipping.md

@@ -147,7 +147,7 @@ Keep these identities distinct:
 | Before: verifier `docker_compose_file` / hash | Raw preserved pre-N0 compose: `df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91` |
 | Before: application `compose_hash` | `bfe9028a58b23238188aa7d70b82282a7cd6414cd83dcc47a3b295c6f64fe2f8`; a distinct reported identity, not the raw-file hash |
 | After: current profile-enabled raw compose | `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`; bind this artifact to the final reviewed PR head before any promotion |
-| After the main release: template-locked raw compose | `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350` (the 1.21.0 compose `d95bfb70…`, template `51f8e9e9…`, with the TC-1274 and TC-1317 regenerated Vector config); node digest normalized, template `7fabba70e71423faa2c8b61981d4a7c95ac7c2552074af81c0877af774d6b383`. See [`docs/releasing.md`](../releasing.md#main-release-with-n0-log-shipping-one-dispatch) |
+| After the main release: template-locked raw compose | `32a002ef5751817619423c9e60776e03f2c3517315f5aa3a9c8654e3394777f1` (the 1.21.0 compose `d95bfb70…`, template `51f8e9e9…`, with the TC-1274, TC-1317 and TC-1482 regenerated Vector config, the three `current_kv_c_index_*` warning codes and the optional TC-1482 `NODE_STATS_READ_TOKEN` env line; before the `current_kv_c_index_*` codes: raw `d1fcd665…`, template `87b5d12a…`; before the TC-1482 changes: raw `1228d819…`, template `7fabba70…`); node digest normalized, template `79469f97a8dcecc7856caa11b26e3004b36d4b8ae9e16a51f963d0687992b9fb`. See [`docs/releasing.md`](../releasing.md#main-release-with-n0-log-shipping-one-dispatch) |
 
 The raw after hash identifies the profile-enabled compose, not deployment approval. Record its final reviewed PR head and required CI evidence in the officer handoff. Do not infer a raw-file hash from the distinct application-reported compose hash.
 

log-shipper/canonical.json

@@ -1,6 +1,6 @@
 {
   "repo": "samgbafa/tinycloud-dev",
-  "revision": "4adaf13b33c85d0a18f63126fc877776c4ee3dbb",
+  "revision": "9bda95fafa167e6255ba7fa85d228e3ee2febdf0",
   "path": "ops/logging/fields.json",
-  "sha256": "02463b9370d88eff282d9c3f5b738f18a169ad6276bacdb9c3d4e60264fb062a"
+  "sha256": "cff33bf50586d8a4425ca1584f125bd891380b74eef76de8e5da690d442ce6d8"
 }