feat: ship redacted tinycloud-node logs to Better Stack

.github/scripts/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "tinycloud-node-deploy-tools",
+  "private": true,
+  "description": "Deploy tooling installed from a lockfile before production secrets are in scope.",
+  "dependencies": {
+    "@phala/cloud": "0.3.0",
+    "phala": "1.1.19"
+  }
+}

.github/scripts/phala-sync-allowed-envs.mjs

@@ -0,0 +1,107 @@
+import { readFileSync } from "node:fs";
+import {
+  createClient,
+  getCvmInfo,
+  getCvmComposeFile,
+  updateCvmEnvs,
+  encryptEnvVars,
+} from "@phala/cloud";
+
+const apiKey = process.env.PHALA_CLOUD_API_KEY;
+const cvmId = process.env.PHALA_CVM_ID;
+const envFile = process.env.ENV_FILE;
+if (!apiKey || !cvmId || !envFile) {
+  console.error("PHALA_CLOUD_API_KEY, PHALA_CVM_ID and ENV_FILE are required");
+  process.exit(1);
+}
+
+const envs = [];
+for (const line of readFileSync(envFile, "utf8").split("\n")) {
+  if (!line.trim() || line.startsWith("#")) continue;
+  const i = line.indexOf("=");
+  if (i <= 0) continue;
+  envs.push({ key: line.slice(0, i), value: line.slice(i + 1) });
+}
+const envKeys = envs.map((e) => e.key);
+if (envKeys.length === 0) {
+  console.error(`No env vars parsed from ${envFile}`);
+  process.exit(1);
+}
+
+const client = createClient({ apiKey });
+const cvmRef = { id: cvmId };
+const compose = await getCvmComposeFile(client, cvmRef, { schema: false });
+const allowed = new Set(compose?.allowed_envs ?? []);
+const nextAllowed = [...new Set([...allowed, ...envKeys])];
+const missing = envKeys.filter((k) => !allowed.has(k));
+if (missing.length === 0) {
+  console.log(`allowed_envs already contains all ${envKeys.length} deploy env keys — nothing to do.`);
+  process.exit(0);
+}
+console.log(`allowed_envs is missing ${missing.length} deploy env key(s): ${missing.join(", ")}`);
+
+const info = await getCvmInfo(client, cvmRef, { schema: false });
+const pubkey = info?.encrypted_env_pubkey ?? info?.kms_info?.encrypted_env_pubkey;
+if (!pubkey) {
+  console.error("Could not resolve encrypted_env_pubkey from CVM info");
+  process.exit(1);
+}
+
+const encryptedEnv = await encryptEnvVars(envs, pubkey);
+let result = await updateCvmEnvs(
+  client,
+  { ...cvmRef, encrypted_env: encryptedEnv, env_keys: nextAllowed },
+  { schema: false },
+);
+console.log("phase 1 response:", JSON.stringify(result));
+
+if (result?.status === "precondition_required") {
+  result = await updateCvmEnvs(
+    client,
+    {
+      ...cvmRef,
+      encrypted_env: encryptedEnv,
+      env_keys: nextAllowed,
+      compose_hash: result.compose_hash,
+    },
+    { schema: false },
+  );
+  console.log("phase 2 response:", JSON.stringify(result));
+}
+
+if (result?.status !== "in_progress") {
+  console.error(`Unexpected env update result: ${JSON.stringify(result)}`);
+  process.exit(1);
+}
+console.log(
+  `Env update started (correlation ${result.correlation_id}, allowed_envs_changed=${result.allowed_envs_changed}). Waiting for CVM to settle...`,
+);
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+let started = false;
+for (let i = 0; i < 120; i++) {
+  await sleep(5000);
+  const cur = await getCvmInfo(client, cvmRef, { schema: false });
+  const status = cur?.status ?? "unknown";
+  if (!started) {
+    if (status !== "running") started = true;
+    else if (i >= 12) started = true;
+    console.log(`[${(i + 1) * 5}s] status=${status} (waiting for update to ${started ? "finish" : "start"})`);
+    continue;
+  }
+  console.log(`[${(i + 1) * 5}s] status=${status}`);
+  if (status === "running") break;
+  if (status === "failed" || status === "error") {
+    console.error(`CVM entered status '${status}' after env update`);
+    process.exit(1);
+  }
+}
+
+const after = await getCvmComposeFile(client, cvmRef, { schema: false });
+const afterAllowed = new Set(after?.allowed_envs ?? []);
+const stillMissing = nextAllowed.filter((k) => !afterAllowed.has(k));
+if (stillMissing.length > 0) {
+  console.error(`allowed_envs still missing after update: ${stillMissing.join(", ")}`);
+  process.exit(1);
+}
+console.log(`allowed_envs now contains all ${envKeys.length} deploy env keys.`);

.github/workflows/docker.yml

@@ -428,6 +428,8 @@ jobs:
       # required for centralized KMS", which broke the 2026-07-28 deploy. The
       # last green deploy (run 29953848846, 2026-07-22) ran on 1.1.19. Only
       # unpin once the prod CVM is migrated or 1.1.20+ is verified against it.
+      - name: Install Phala SDK helpers
+        run: npm ci --prefix .github/scripts --ignore-scripts
       - name: Install Phala CLI
         run: npm install -g phala@1.1.19
 
@@ -598,6 +600,7 @@ jobs:
           PHALA_ENV_CLOUDFLARE_API_TOKEN: ${{ secrets.PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN }}
           PHALA_ENV_DSTACK_GATEWAY_DOMAIN: ${{ secrets.PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN }}
           PHALA_ENV_CERTBOT_EMAIL: ${{ secrets.PROD_TINYCLOUD_CERTBOT_EMAIL }}
+          PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }}
           PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }}
         # Update the existing prod CVM (name: tinycloud-node) in the
         # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is
@@ -642,6 +645,7 @@ jobs:
             printf 'CLOUDFLARE_API_TOKEN=%s\n' "$PHALA_ENV_CLOUDFLARE_API_TOKEN"
             printf 'DSTACK_GATEWAY_DOMAIN=%s\n' "$PHALA_ENV_DSTACK_GATEWAY_DOMAIN"
             printf 'CERTBOT_EMAIL=%s\n' "$PHALA_ENV_CERTBOT_EMAIL"
+            printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN"
             printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE"
             # SSH public keys authorized on the CVM (checked into the repo,
             # see dstack-authorized-keys). The Phala Cloud pre-launch script
@@ -657,6 +661,13 @@ jobs:
             -f docker-compose.dstack-postgres.yaml config --quiet
 
           phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait
+      # Run only after the compose deploy so a Phala allowed_envs sync failure cannot gate node startup.
+      - name: Sync production allowed_envs
+        env:
+          PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }}
+          PHALA_CVM_ID: tinycloud-node
+          ENV_FILE: .phala-prod.env
+        run: node .github/scripts/phala-sync-allowed-envs.mjs
 
       # TC-312: prove the deploy actually converged. The image tag matching the
       # crate version is enforced at build time by version-guard, but nothing

.github/workflows/rust.yml

@@ -187,6 +187,34 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: 20
+    - name: Check generated log allowlist and compose parity
+      run: |
+        npm ci --ignore-scripts --no-audit --no-fund
+        node scripts/node-vector-schema.mjs --check
+        node scripts/node-vector-schema.mjs --check-compose
+        node --test scripts/node-vector-schema.test.mjs scripts/tc-959-vector-metrics.test.mjs scripts/tc-959-compose-env.test.mjs
+        bash scripts/test-tc-959-isolation.sh
+    - name: Test Vector allowlist (0.59.0, checksum pinned)
+      run: |
+        set -eu
+        version=0.59.0
+        archive="vector-${version}-x86_64-unknown-linux-musl.tar.gz"
+        expected_sha256=a8dbc43c18ae25d0b23a712c9262f3aad904e352b1db5e7db1d9c4aecab59496
+        release_url="https://github.com/vectordotdev/vector/releases/download/v${version}"
+        curl -fsSL "${release_url}/${archive}" -o "${RUNNER_TEMP}/${archive}"
+        curl -fsSL "${release_url}/vector-${version}-SHA256SUMS" -o "${RUNNER_TEMP}/vector-${version}-SHA256SUMS"
+        printf '%s  %s\n' "${expected_sha256}" "${archive}" | (cd "${RUNNER_TEMP}" && sha256sum --check)
+        (cd "${RUNNER_TEMP}" && sha256sum --check --ignore-missing "vector-${version}-SHA256SUMS")
+        mkdir -p "${RUNNER_TEMP}/vector"
+        tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}/vector"
+        vector="${RUNNER_TEMP}/vector/vector-x86_64-unknown-linux-musl/bin/vector"
+        sudo install -d -o "$(id -u)" -g "$(id -g)" -m 700 /var/lib/vector/secrets
+        trap 'sudo rm -rf /var/lib/vector' EXIT
+        printf '%s' 'dummy-vector-test-token' > /var/lib/vector/secrets/bs_token
+        chmod 600 /var/lib/vector/secrets/bs_token
+        "${vector}" validate --no-environment log-shipper/vector.yaml
+        "${vector}" test log-shipper/vector.yaml log-shipper/vector.tests.yaml
+        TC_VECTOR_BIN="${vector}" TC_DRY_RUN=1 bash ./tc-959-node-docker-check.review.sh
     - name: Verify generated capability artifacts match capabilities.json
       run: node scripts/gen-capabilities.mjs --check
     - name: Verify deployed Policy/v3 probe contract

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## [Unreleased]
+- TC-959: ship only labelled node/ingress logs through GELF/UDP to a fixed-address, isolated Vector socket; cap complete node JSON lines at 15,000 bytes including newline, with the panic hook on the same path; gate labels before metadata stripping, JSON parsing, and the existing allowlist. Gate reasons are summarized without content on an independent 1/min lane; ingress is capped at 26/min to preserve the 39/min, 0.473 GB/30d budget while app WARN remains 5/min. Vector publishes UDP on host loopback only, has no Docker-directory mount or socket, and is capped at 64 MiB and 0.5 CPU on the 2-vCPU CVM. Unlabelled services remain on capped `json-file`.
 
 - Security (TC-1132, TC-1080 Parts C–D): webhook registration and every delivery attempt share an egress policy covering special-use IPv4/IPv6, embedded IPv4 (mapped, compatible, NAT64 and 6to4), Teredo and site-local IPv6. Ambiguous local-use NAT64 is refused if any valid IPv4 interpretation is forbidden. Corrected public controls such as `192.2.1.1` and `198.51.0.1` are no longer over-blocked. Mixed DNS answers retain only permitted addresses.
 - Webhooks deny the resolved addresses of the configured network database, quota URL, space allowlist and explicit S3 endpoint on every port, including aliases and embedded IPv4 forms. The service deny-set refreshes every ten minutes; validation waits for an in-progress refresh, and failed/stale resolution closes egress until recovery. Temporary DNS/policy failures retry on the existing bounded schedule; invalid or forbidden callbacks dead-letter. Existing subscriptions whose destinations become forbidden dead-letter on the next attempt.

README.md

@@ -304,6 +304,16 @@ TINYCLOUD_PORT=8001 tinycloud
 
 If the TinyCloud Protocol instance is not able to find or establish a connection to the configured storage, the instance will terminate.
 
+### Production log shipping (TC-959)
+
+The node and ingress use the Docker GELF driver with exact `tc.logship` labels and send UDP to host loopback `127.0.0.1:12201`. Vector runs on its own fixed-address bridge network, publishes only that loopback UDP port, and has no Docker socket or host Docker-directory mount. Its first transform gates on `_tc.logship`; only then does it strip GELF metadata and parse the app's JSON line before the existing schema allowlist. Other services, including Postgres, stay unlabelled on capped `json-file` logs. Node and ingress do not depend on Vector; a missing Better Stack token leaves Vector idle. The node JSON formatter and panic hook cap each complete line, including newline, at 15,000 bytes; ingress's upstream status/time lists have no explicit size bound in the checked config.
+
+The panic, app ERROR, app WARN, gate-summary, and ingress lanes are capped at 2, 5, 5, 1, and 26 events/minute. The one-per-minute gate summary carries four reason counts and no content; it cannot consume a real WARN slot. With the 281-byte final event limit, the monthly ceiling is `(2 + 5 + 5 + 1 + 26) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), below the node's 0.48 GB share of the 3 GB free quota. Unknown routes collapse to `<unmatched>`.
+
+On the 2-vCPU/4-GiB/20-GB tdx.medium CVM, Vector is capped at 64 MiB and ≤0.5 CPU (25% of the CVM's 2 vCPU). Its data directory and token file live on an 8 MiB tmpfs; the Vector and ssh-gateway `json-file` logs are capped at `10m × 3` each (60 MB total). The review-only Docker acceptance script measures Vector peak memory and transport loss; neither has been measured by this change.
+
+The production token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint copies it to a mode-0600 file on Vector's tmpfs, then unsets the token and executes Vector. Production uses `unset …; exec vector`; `exec env -u` exists only in the acceptance override. The token is not in Vector's process environment. It remains in the container's configured environment (visible to `docker inspect`) and in the init process's initial environment, which only principals that can already read the tmpfs token file can read. Restrict access to both; see [`docs/runbooks/tc-959-betterstack-log-shipping.md`](docs/runbooks/tc-959-betterstack-log-shipping.md) for details.
+
 ## Usage
 
 TinyCloud Protocol is most easily used via the TinyCloud Protocol SDK. See the example DApps and tutorials for detailed information.