logging: log /admin/stats as its own route, status only (TC-1482)

.github/workflows/docker.yml

@@ -666,10 +666,10 @@ jobs:
           native = 'ghcr.io/tinycloudlabs/tinycloud-node'
           reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f'
           # TC-959: main's N0 compose (#349 c33c2302…) with the 95m ingress body
-          # cap (raw d95bfb70…), the TC-1274 + TC-1317 Vector config and the
-          # optional TC-1482 NODE_STATS_READ_TOKEN env line (raw 8f67b3b2…),
+          # cap (raw d95bfb70…), the TC-1274 + TC-1317 + TC-1482 + c_index Vector config and the
+          # optional TC-1482 NODE_STATS_READ_TOKEN env line (raw 32a002ef…),
           # its single node digest normalized to 64 zeros.
-          n0_template_hash = 'ae497fed720da12219999f4bb8573fb81b235c25086c96142e87f11e948b5832'
+          n0_template_hash = '79469f97a8dcecc7856caa11b26e3004b36d4b8ae9e16a51f963d0687992b9fb'
           node_pin = re.compile(rb'(?m)^    image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$')
           def n0_template(raw):
               pins = list(node_pin.finditer(raw))

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## [Unreleased]
 
+### Changes
+
+- `/admin/stats` is a logged route label (TC-1482). The log shipper and the ingress log it as the route template plus status only, with no query string, `Authorization` header, request or response body, or client address (TC-961).
+  - **N0 compose.** The embedded Vector configuration is regenerated for the new route. Together with the optional `NODE_STATS_READ_TOKEN` env line, the reviewed N0 compose template changes to `79469f97…` (raw `32a002ef…`; `docs/releasing.md`).
+  - **`current_kv` C-index warning codes.** The log sidecar maps the three constant warnings of the TC-1274 `current_kv` C-index migration to `current_kv_c_index_missing`, `current_kv_c_index_invalid` and `current_kv_c_index_wrong_shape` (TC-1274); until then they would ship as `unclassified`. This is part of the same regenerated Vector configuration.
+  - **Ingress.** The `/admin/stats` label takes effect once the ingress image is rebuilt from `deploy/ingress` and its digest is pinned in the compose; until then the deployed ingress logs it as `<other>`.
+
 ## [1.21.1] - 2026-10-19
 
 ### Release notes

deploy/ingress/redacted-logging.conf

@@ -24,6 +24,7 @@ map $uri $tc_route {
     /signed/kv                                      /signed/kv;
     /admin/quota                                    /admin/quota;
     /admin/usage                                    /admin/usage;
+    /admin/stats                                    /admin/stats;
     /hooks/events                                   /hooks/events;
     /hooks/tickets                                  /hooks/tickets;
     /hooks/webhooks                                 /hooks/webhooks;

deploy/ingress/test-redacted-logging.sh

@@ -43,6 +43,7 @@ echo "testing $IMAGE"
 hex() { od -An -N"$1" -tx1 /dev/urandom | tr -d ' \n'; }
 TICKET="tc970ticket$(hex 12)"
 HOOK_TICKET="tc970hookticket$(hex 12)"
+STATS_TOKEN="tc1482statstoken$(hex 12)"
 WALLET="$(hex 20)"
 SPACE="tinycloud:pkh:eip155:1:0x${WALLET}:default"
 KEY="tc970key$(hex 12)"
@@ -128,6 +129,8 @@ expect_status() {
 
 expect_status 401 "https://$DOMAIN/signed/kv/$TICKET"
 expect_status 200 "https://$DOMAIN/hooks/events?ticket=$HOOK_TICKET"
+# TC-1482: /admin/stats is its own route; its query string never reaches the log.
+expect_status 200 "https://$DOMAIN/admin/stats?token=$STATS_TOKEN"
 expect_status 502 "https://$DOMAIN/public/$SPACE/kv/$KEY"
 expect_status 401 "https://$DOMAIN/signed/kv/$TICKET/unmatched"
 # HTTP/1.1, a client-chosen method, and a 400 raised before routing (a header
@@ -156,6 +159,7 @@ FILES=$($DOCKER exec "$INGRESS" sh -c 'find /var/log/nginx -type f -exec cat {}
 for pair in \
     "signed-KV ticket=$TICKET" \
     "hook ticket=$HOOK_TICKET" \
+    "admin stats query token=$STATS_TOKEN" \
     "space id=$SPACE" \
     "wallet address=$WALLET" \
     "KV key=$KEY" \
@@ -200,6 +204,7 @@ expect_line() {
 }
 expect_line GET "/signed/kv/<ticket_id>" 401
 expect_line GET "/hooks/events" 200
+expect_line GET "/admin/stats" 200
 expect_line GET "/public/<space_id>/kv/<key..>" 502
 expect_line GET "<other>" 401
 expect_line "<other>" "/signed/kv/<ticket_id>" 401

docs/releasing.md

@@ -370,11 +370,11 @@ Before any CVM mutation the workflow requires:
 
 - the main compose, with its one node digest replaced by 64 zeros, to hash
   to the reviewed N0 template
-  **`ae497fed720da12219999f4bb8573fb81b235c25086c96142e87f11e948b5832`**
-  (raw `8f67b3b25136537c65b154b8729b0a1a7d9911b9963acc121a303cb9d806a6e7` normalized; before the optional TC-1482 `NODE_STATS_READ_TOKEN` env line the template was `7fabba70…`, raw `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350`: the 1.21.0 compose
+  **`79469f97a8dcecc7856caa11b26e3004b36d4b8ae9e16a51f963d0687992b9fb`**
+  (raw `32a002ef5751817619423c9e60776e03f2c3517315f5aa3a9c8654e3394777f1` normalized; before the three `current_kv_c_index_*` warning codes in the Vector config (TC-1274 `current_kv` C-index migration) the template was `87b5d12a…`, raw `d1fcd665…`; before the optional TC-1482 `NODE_STATS_READ_TOKEN` env line and the TC-1482 route in the Vector config the template was `7fabba70…`, raw `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350`: the 1.21.0 compose
   `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50`, template
-  `51f8e9e9…`, with only the embedded Vector config regenerated for TC-1274's
-  and TC-1317's warning codes; TC-1274 alone was template `00938226…`, raw
+  `51f8e9e9…`, with only the embedded Vector config regenerated for TC-1274's,
+  TC-1317's and TC-1482's route and warning codes (TC-1274 and TC-1317 were template `7fabba70…`, raw `1228d819…`); TC-1274 alone was template `00938226…`, raw
   `b25b8414…`. That compose is the #349 compose
   `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`, which
   keeps `TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"`, with only
@@ -440,8 +440,8 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \
 
 The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**.
 With the template lock on main this redeploys exactly main's raw compose
-`8f67b3b25136537c65b154b8729b0a1a7d9911b9963acc121a303cb9d806a6e7`
-(the 1.21.0 compose `d95bfb70…` with the TC-1274 and TC-1317 Vector config and the optional TC-1482 `NODE_STATS_READ_TOKEN` env line), whatever digest
+`32a002ef5751817619423c9e60776e03f2c3517315f5aa3a9c8654e3394777f1`
+(the 1.21.0 compose `d95bfb70…` with the TC-1274, TC-1317, TC-1482 and `current_kv_c_index_*` Vector config and the optional TC-1482 `NODE_STATS_READ_TOKEN` env line), whatever digest
 main's template currently pins. It is not a rollback of a
 later release across a schema migration. Current pre-N0 tag/pinned
 equivalents are separately hash-checked; arbitrary same-image config

docs/runbooks/tc-959-n0-log-shipping.md

@@ -147,7 +147,7 @@ Keep these identities distinct:
 | Before: verifier `docker_compose_file` / hash | Raw preserved pre-N0 compose: `df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91` |
 | Before: application `compose_hash` | `bfe9028a58b23238188aa7d70b82282a7cd6414cd83dcc47a3b295c6f64fe2f8`; a distinct reported identity, not the raw-file hash |
 | After: current profile-enabled raw compose | `c33c23021589163b62a71ef245b7194717711922027629f155bdd5cff806b8c7`; bind this artifact to the final reviewed PR head before any promotion |
-| After the main release: template-locked raw compose | `8f67b3b25136537c65b154b8729b0a1a7d9911b9963acc121a303cb9d806a6e7` (the 1.21.0 compose `d95bfb70…`, template `51f8e9e9…`, with the TC-1274 and TC-1317 regenerated Vector config and the optional TC-1482 `NODE_STATS_READ_TOKEN` env line; before that line: raw `1228d819…`, template `7fabba70…`); node digest normalized, template `ae497fed720da12219999f4bb8573fb81b235c25086c96142e87f11e948b5832`. See [`docs/releasing.md`](../releasing.md#main-release-with-n0-log-shipping-one-dispatch) |
+| After the main release: template-locked raw compose | `32a002ef5751817619423c9e60776e03f2c3517315f5aa3a9c8654e3394777f1` (the 1.21.0 compose `d95bfb70…`, template `51f8e9e9…`, with the TC-1274, TC-1317 and TC-1482 regenerated Vector config, the three `current_kv_c_index_*` warning codes and the optional TC-1482 `NODE_STATS_READ_TOKEN` env line; before the `current_kv_c_index_*` codes: raw `d1fcd665…`, template `87b5d12a…`; before the TC-1482 changes: raw `1228d819…`, template `7fabba70…`); node digest normalized, template `79469f97a8dcecc7856caa11b26e3004b36d4b8ae9e16a51f963d0687992b9fb`. See [`docs/releasing.md`](../releasing.md#main-release-with-n0-log-shipping-one-dispatch) |
 
 The raw after hash identifies the profile-enabled compose, not deployment approval. Record its final reviewed PR head and required CI evidence in the officer handoff. Do not infer a raw-file hash from the distinct application-reported compose hash.
 

log-shipper/canonical.json

@@ -1,6 +1,6 @@
 {
   "repo": "samgbafa/tinycloud-dev",
-  "revision": "4adaf13b33c85d0a18f63126fc877776c4ee3dbb",
+  "revision": "ee823cd5e2aed6ece73bc80ce5123793f67a0e99",
   "path": "ops/logging/fields.json",
-  "sha256": "02463b9370d88eff282d9c3f5b738f18a169ad6276bacdb9c3d4e60264fb062a"
+  "sha256": "cff33bf50586d8a4425ca1584f125bd891380b74eef76de8e5da690d442ce6d8"
 }

log-shipper/cases.mjs

@@ -8,6 +8,12 @@ const safe = '!contains(encode_json(.), "TC959_SYNTHETIC") && !exists(.message)
 // Vector test log_fields keys are paths; quote the literal dotted GELF label.
 const logFields = input => Object.fromEntries(Object.entries(input).map(([key, value]) => [key === '_tc.logship' ? '."_tc.logship"' : key, value]));
 const check = (name, input, assertion, insertAt = 'logship_gate') => ({ name, inputs: [{ insert_at: insertAt, type: 'log', log_fields: logFields(input) }], outputs: [{ extract_from: 'allowlist', conditions: [{ type: 'vrl', source: `assert!(${assertion})\nassert!(${safe})` }] }] });
+// Constant messages emitted by tinycloud-core migration m20261020_000000_current_kv_key_c_index (TC-1274); production logs drop the `event` field, so the exact text is the only key.
+const currentKvCIndexMessages = {
+  "missing": "idx_current_kv_space_deleted_key_c is missing; the node keeps serving and KV list requests are slower until an operator builds it with CREATE INDEX CONCURRENTLY (docs/releasing.md, section idx_current_kv_space_deleted_key_c)",
+  "invalid": "idx_current_kv_space_deleted_key_c is invalid or still being built; the node keeps serving and KV list requests are slower. If no CREATE INDEX CONCURRENTLY is running, run DROP INDEX CONCURRENTLY idx_current_kv_space_deleted_key_c; and then the CREATE INDEX CONCURRENTLY in docs/releasing.md, section idx_current_kv_space_deleted_key_c",
+  "wrong_shape": "idx_current_kv_space_deleted_key_c exists with the wrong definition and is not used for KV lists; the node keeps serving and KV list requests are slower. Run DROP INDEX CONCURRENTLY idx_current_kv_space_deleted_key_c; and then the CREATE INDEX CONCURRENTLY in docs/releasing.md, section idx_current_kv_space_deleted_key_c"
+};
 // Like check(), but the output may carry the validated N2 correlation fields; nothing else may leak.
 const unsafeCorrelated = '!contains(encode_json(.), "TC959_SYNTHETIC") && !exists(.message) && !exists(.target) && !exists(.host)';
 const correlated = (name, input, assertion) => ({ name, inputs: [{ insert_at: 'logship_gate', type: 'log', log_fields: logFields(input) }], outputs: [{ extract_from: 'allowlist', conditions: [{ type: 'vrl', source: `assert!(${assertion})\nassert!(${unsafeCorrelated})` }] }] });
@@ -25,6 +31,10 @@ export const vectorTests = [
   check('long latency is not dropped by byte budget', envelope(nodeRequest({ route: '/policy/v3/deliveries/authorize', latency_us: 9007199254740991 })), '.latency_us == 9007199254740991 && .event == "node_request"'),
   ...['TRACE', 'CONNECT'].map(method => check(`${method} request uses the bounded other method`, envelope(nodeRequest({ method, route: '<unmatched>', status: 404 })), '.event == "node_request" && .method == "<other>" && .status == 404')),
   check('unknown extras are discarded', envelope({ ...nodeRequest({ authorization: canary, nested: { sql: canary }, unexpected: canary }), span: { account: canary }, spans: [{ request_id: canary }] }), '.event == "node_request" && !exists(.authorization) && !exists(.nested) && !exists(.unexpected) && !exists(.span)'),
+  // TC-1482: /admin/stats is its own route label; only the route template and status ship.
+  check('node /admin/stats ships as route plus status only', envelope(nodeRequest({ method: 'GET', route: '/admin/stats', status: 401, authorization: `Bearer ${canary}`, query: canary, uri: `/admin/stats?token=${canary}`, body: canary, address: canary })), '.event == "node_request" && .method == "GET" && .route == "/admin/stats" && .status == 401 && keys!(.) == ["event", "latency_us", "level", "method", "route", "status", "svc", "ts"]'),
+  check('ingress /admin/stats ships as route plus status only', envelope(ingressRequest({ method: 'GET', route: '/admin/stats', status: 503, authorization: `Bearer ${canary}`, query: canary, uri: `/admin/stats?token=${canary}`, upstream_status: canary, body: canary, address: canary }), 'tinycloud-ingress'), '.event == "ingress_request" && .method == "GET" && .route == "/admin/stats" && .status == 503 && keys!(.) == ["event", "level", "method", "request_time_seconds", "route", "status", "svc", "ts"]'),
+  reject('a query string or address in the route is rejected, never shipped', envelope(nodeRequest({ method: 'GET', route: `/admin/stats?token=${canary}`, status: 401 }))),
   correlated('valid N2 correlation fields pass through exactly', envelope({ ...nodeRequest({ request_id: goodId, request_id_replaced: true, account_hash: goodHash, space_hash: goodSpace }), span: { account: canary }, spans: [{ request_id: canary }] }), `.event == "node_request" && .request_id == "${goodId}" && .request_id_replaced == true && .account_hash == "${goodHash}" && .space_hash == "${goodSpace}" && !exists(.span) && !exists(.spans)`),
   correlated('absent hashes (no space on the request) stay absent', envelope(nodeRequest({ request_id: goodId, request_id_replaced: false })), `.event == "node_request" && .request_id == "${goodId}" && .request_id_replaced == false && !exists(.account_hash) && !exists(.space_hash)`),
   correlated('malformed correlation values are dropped but the request ships', envelope(nodeRequest({ request_id: 'not-a-uuid-TC959_SYNTHETIC', request_id_replaced: 'yes', account_hash: 'A'.repeat(64), space_hash: 'a'.repeat(63) })), '.event == "node_request" && .route == "/invoke" && !exists(.request_id) && !exists(.request_id_replaced) && !exists(.account_hash) && !exists(.space_hash)'),
@@ -44,6 +54,7 @@ export const vectorTests = [
   ...[['Request deadline exceeded', 'request_timeout'], ['Request completed after its deadline', 'deadline_overrun']].map(([message, code]) => check(`request deadline warning maps to ${code} without its fields`, envelope({ timestamp, level: 'WARN', target: 'tinycloud_core::request_cancel', fields: { message, error_class: canary, stage: canary, deadline_ms: 55000 } }), `.event == "node_warning" && .event_code == "${code}" && .event_count == 1`)),
   check('hydration timeout warning maps to sql_hydration_timeout without its fields', envelope({ timestamp, level: 'WARN', target: 'tinycloud_core::sql::service', fields: { message: 'Database hydration gave up loading its artifact', service: 'sql', error_class: canary, bound_ms: 600000 } }), '.event == "node_warning" && .event_code == "sql_hydration_timeout" && .event_count == 1'),
   ...[['Read audit wait expired', 'read_audit_busy'], ['Read audit batch slow', 'read_audit_batch_slow'], ['Read audit batch failed', 'read_audit_batch_failed'], ['Read audit drain stalled', 'read_audit_drain_stalled']].map(([message, code]) => check(`read-audit warning maps to ${code} without its fields`, envelope({ timestamp, level: 'WARN', target: 'tinycloud_core::read_audit', fields: { message, error_class: canary, count: 3, queue_depth: 1024, in_flight_batch_age_ms: 30001, abandoned: 7 } }), `.event == "node_warning" && .event_code == "${code}" && .event_count == 1`)),
+  ...[['missing', 'WARN', 'node_warning', 'current_kv_c_index_missing'], ['invalid', 'WARN', 'node_warning', 'current_kv_c_index_invalid'], ['wrong_shape', 'ERROR', 'node_error', 'current_kv_c_index_wrong_shape']].map(([key, level, event, code]) => check(`current_kv C index ${key} message maps to ${code} with only the fixed diagnostic keys`, envelope({ timestamp, level, target: 'tinycloud_core::migrations::m20261020_000000_current_kv_key_c_index', fields: { message: currentKvCIndexMessages[key], event: canary, index: canary, reason: canary } }), `.event == "${event}" && .event_code == "${code}" && .event_count == 1 && .level == "${level.toLowerCase()}" && keys(.) == ["event", "event_code", "event_count", "level", "svc", "ts"]`)),
   check('dependency warning is a fixed unclassified diagnostic', envelope({ timestamp, level: 'WARN', target: 'hyper::proto', fields: { message: canary, error: canary } }), '.event == "node_warning" && .event_code == "unclassified" && .event_count == 1'),
   check('dependency error without a message still counts', envelope({ timestamp, level: 'ERROR', target: 'sqlx::pool', fields: { error: canary } }), '.event == "node_error" && .event_code == "unclassified" && .event_count == 1'),
   check('warning without a target or fields still counts', envelope({ timestamp, level: 'WARN' }), '.event == "node_warning" && .event_code == "unclassified" && .event_count == 1'),

log-shipper/diagnostics.json

@@ -57,7 +57,10 @@
     "Read audit wait expired": "read_audit_busy",
     "Read audit batch slow": "read_audit_batch_slow",
     "Read audit batch failed": "read_audit_batch_failed",
-    "Read audit drain stalled": "read_audit_drain_stalled"
+    "Read audit drain stalled": "read_audit_drain_stalled",
+    "idx_current_kv_space_deleted_key_c is missing; the node keeps serving and KV list requests are slower until an operator builds it with CREATE INDEX CONCURRENTLY (docs/releasing.md, section idx_current_kv_space_deleted_key_c)": "current_kv_c_index_missing",
+    "idx_current_kv_space_deleted_key_c is invalid or still being built; the node keeps serving and KV list requests are slower. If no CREATE INDEX CONCURRENTLY is running, run DROP INDEX CONCURRENTLY idx_current_kv_space_deleted_key_c; and then the CREATE INDEX CONCURRENTLY in docs/releasing.md, section idx_current_kv_space_deleted_key_c": "current_kv_c_index_invalid",
+    "idx_current_kv_space_deleted_key_c exists with the wrong definition and is not used for KV lists; the node keeps serving and KV list requests are slower. Run DROP INDEX CONCURRENTLY idx_current_kv_space_deleted_key_c; and then the CREATE INDEX CONCURRENTLY in docs/releasing.md, section idx_current_kv_space_deleted_key_c": "current_kv_c_index_wrong_shape"
   },
   "prefix": {
     "dstack socket available but get_info failed: ": "dstack_get_info_failed",