From 126adcba8c0426ad6087e7eded6d8b2e674f1be9 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 08:30:25 +0000 Subject: [PATCH 1/4] ci(TC-887): allow manual dispatch of the Rust workflow for stacked branches --- .github/workflows/rust.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fdffea5c..e2ac72d5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -5,6 +5,9 @@ on: branches: [ main ] pull_request: branches: [ main ] + # TC-887: run the full Rust CI on a stacked branch (a PR whose base is not + # main gets no pull_request run). Manual only; same jobs as a PR to main. + workflow_dispatch: env: CARGO_TERM_COLOR: always From c51429cc4e432aa87726182ea845b6588f13333f Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 08:34:54 +0000 Subject: [PATCH 2/4] docs(TC-887): put the at-rest interfaces in the repo; one generation ceiling docs/at-rest/INTERFACES.md carries the normative TCE1 format, the TCE2 versioning rule, the fail-closed and header-trust notes and the KV storage decisions; the module docs now point to it. PayloadEncoding reads 2..=9 as an unsupported later generation (the range the header can express) and anything above 9 as invalid. --- docs/at-rest/INTERFACES.md | 292 +++++++++++++++++++++++++ tinycloud-core/src/at_rest/envelope.rs | 5 +- tinycloud-core/src/at_rest/names.rs | 9 +- tinycloud-core/src/at_rest/provider.rs | 2 +- tinycloud-core/src/at_rest/tests.rs | 4 +- 5 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 docs/at-rest/INTERFACES.md diff --git a/docs/at-rest/INTERFACES.md b/docs/at-rest/INTERFACES.md new file mode 100644 index 00000000..77ad1acf --- /dev/null +++ b/docs/at-rest/INTERFACES.md @@ -0,0 +1,292 @@ +# Encryption at rest: interfaces (TC-887) + +Normative interface and format for `tinycloud_core::at_rest` and the node's sealed KV block +storage. Plan: TC-887 (encryption at rest inside the TEE). Key continuity and escrow: +`docs/at-rest-escrow.md`. + +Responsibilities: +- **Key continuity** (`at_rest::keyring`, node `at_rest` wiring) owns the root → wrap key → + at-rest root key → KEK/NK hierarchy, data-key (DEK) creation, wrap/unwrap and caching, the + key tables, escrow, the encrypt gate, locked mode and the `off`/`encrypt`/`strict` mode + config. It implements `DataKeyProvider`. +- **The envelope** (`at_rest::{envelope,stream,names}`) owns everything below the DEK: per-blob + key derivation, the TCE1 format, AAD, segmenting, streaming, Range mapping, object names + from NK, and the payload-encoding flag. It takes a `DataKey` value and never calls the + provider. +- **Storage** (KV block stores; SQL artifacts) glues them: `provider.current_key(space)` → + seal; parse header → `provider.key(space, header.dek_id())` → open. + +## 1. Key types + +```rust +/// Per-space data-key id. Allocated by the key store, unique per space (not global). +/// Stored in the TCE1 header as u32 BE. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DekId(u32); +impl DekId { pub const fn new(v: u32) -> Self; pub const fn get(self) -> u32; } + +/// An unwrapped 32-byte DEK plus its id. Zeroized on drop; Debug prints only the id. +#[derive(Clone)] +pub struct DataKey { /* id: DekId, bytes: Zeroizing<[u8; 32]> */ } +impl DataKey { + pub fn new(id: DekId, bytes: [u8; 32]) -> Self; + pub fn id(&self) -> DekId; + // no public accessor for the bytes +} + +/// NK = HKDF(ARK_1, "object-names/v1"), pinned to ARK generation 1 (TC-887 plan §1.1). +/// Zeroized on drop; Debug redacted. +#[derive(Clone)] +pub struct ObjectNameKey { /* Zeroizing<[u8; 32]> */ } +impl ObjectNameKey { pub fn new(bytes: [u8; 32]) -> Self; } +``` + +Space identifiers everywhere in this API are `&str` in canonical `SpaceId::to_string()` +form (what PG stores; SQL repo already has `&str`; KV callers pass `space.to_string()`). + +## 2. `DataKeyProvider` + +```rust +#[async_trait::async_trait] +pub trait DataKeyProvider: Send + Sync { + /// The DEK new writes in `space` must use. If the space has none, create one + /// (32 random bytes), wrap it under the current KEK, and store it durably + /// BEFORE returning. Concurrent first calls (incl. across processes) must + /// converge on one dek_id (e.g. INSERT … ON CONFLICT DO NOTHING, then re-read). + async fn current_key(&self, space: &str) -> Result; + + /// The DEK a stored blob names in its header. Never creates. Must work for + /// every dek_id ever issued for `space` (old DEKs are never deleted except + /// by deliberate crypto-shred). + async fn key(&self, space: &str, dek_id: DekId) -> Result; + + /// NK for sealed S3 object names. Same value for the life of the deployment. + fn object_name_key(&self) -> Result; +} + +#[derive(Debug, thiserror::Error)] +pub enum DataKeyError { + /// Node booted locked (W cannot unseal ARK). Callers map to HTTP 503. + #[error("at-rest keys are locked")] + Locked, + /// Header names a dek_id this space never had (or it was shredded). + /// Callers map to an integrity error, never to "not found". + #[error("unknown data key {dek_id:?} for space")] + UnknownKey { dek_id: DekId }, + /// PG/S3/unwrap failure (incl. a wrapped DEK failing AEAD). Opaque, no key bytes. + #[error("data key backend: {0}")] + Backend(#[source] Box), +} +``` + +## 3. Envelope API + +```rust +pub const HEADER_LEN: usize = 64; +pub const SEGMENT_LEN: usize = 1 << 16; // plaintext bytes per segment +pub const TAG_LEN: usize = 16; + +pub enum BlobIdentity<'a> { // what the ciphertext is bound to (§4) + KvBlock { space: &'a str, content_hash: &'a Hash }, + SqlCheckpoint { service: &'a str, space: &'a str, name: &'a str, + checkpoint_content_hash: &'a str, checkpoint_size_bytes: u64 }, + SqlWal { service: &'a str, space: &'a str, name: &'a str, revision: i64, + checkpoint_content_hash: &'a str, delta_content_hash: &'a str }, +} + +/// Parsed, validated 64-byte header (unknown version/alg/flags/padding refused). +pub struct Header { /* raw bytes, dek_id, plaintext_len */ } +impl Header { + pub fn parse(bytes: &[u8]) -> Result; // reads the first 64 bytes + pub fn as_bytes(&self) -> &[u8; 64]; + pub fn dek_id(&self) -> DekId; + pub fn plaintext_len(&self) -> u64; + pub fn ciphertext_len(&self) -> u64; + /// Resolves `spec` against the PLAINTEXT length and maps it onto whole segments. + /// None = unsatisfiable (answer 416 with plaintext_len() as total). + pub fn map_range(&self, spec: ByteRangeSpec) -> Option; +} + +pub fn ciphertext_len(plaintext_len: u64) -> Result; +/// Inverse, for the boot size scan: None if `ct_len` is not a valid TCE1 length. +pub fn plaintext_len(ciphertext_len: u64) -> Option; + +/// Whole-buffer seal/open, in place (no extra plaintext-sized allocation). +pub fn seal(key: &DataKey, id: &BlobIdentity<'_>, plaintext: Vec) -> Result, EnvelopeError>; +/// Fails unless header.dek_id == key.id(). Never returns partial plaintext. +pub fn open(key: &DataKey, id: &BlobIdentity<'_>, ciphertext: Vec) -> Result, EnvelopeError>; + +/// Streaming (futures::io::AsyncRead, matching storage::Content). `R: Unpin` +/// (wrap with Box::pin). Each holds at most one segment in memory. +pub struct EncryptReader; // plaintext R of declared length -> header ‖ segments +impl EncryptReader { + /// Errors (io InvalidData, inner LengthMismatch) if R yields fewer/more bytes than declared. + pub fn new(key: &DataKey, id: &BlobIdentity<'_>, plaintext_len: u64, inner: R) -> Result; +} +pub struct DecryptReader; // ciphertext R -> verified plaintext, one segment at a time +impl DecryptReader { + /// `inner` positioned just after the header. + pub fn new(key: &DataKey, id: &BlobIdentity<'_>, header: &Header, inner: R) -> Result; + /// `inner` yields exactly bytes `range.ciphertext_start()..=range.ciphertext_end()`. + /// Refuses a SegmentRange mapped from a different header. + pub fn range(key: &DataKey, id: &BlobIdentity<'_>, header: &Header, + range: SegmentRange, inner: R) -> Result; +} +/// A mid-stream io::Error (kind InvalidData, inner EnvelopeError::Integrity) means the +/// object is corrupt/truncated; already-yielded bytes were authentic but the response +/// must be aborted, not completed. Errors are sticky (the reader is poisoned). + +pub struct SegmentRange { /* opaque */ } +impl SegmentRange { + pub fn ciphertext_start(&self) -> u64; // inclusive, absolute offset in object + pub fn ciphertext_end(&self) -> u64; // inclusive + pub fn plaintext_range(&self) -> ResolvedByteRange; // for Content-Range + pub fn total_size(&self) -> u64; // plaintext total +} + +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] // DataKeyError too +pub enum EnvelopeError { + Header(&'static str), // malformed / not TCE: magic, version-byte mismatch, alg, flags, padding, seg size, length; invalid PayloadEncoding (≤0, >9) + Unsupported { version: u8 }, // well-formed later generation (TCE2+), or PayloadEncoding 2..=9: "upgrade", not corruption + KeyMismatch, // header dek_id != key.id() + Integrity, // any AEAD failure, truncation, reorder, trailing bytes, foreign identity + TooLarge, // > 2^32 segments + LengthMismatch, // EncryptReader input length != declared +} +/// First-5-bytes dispatch: "TCE" ‖ ASCII digit g ‖ byte g → Ok(g); else Header. +pub fn format_version(bytes: &[u8]) -> Result; +``` + +Memory: each reader holds two (64 KiB + 16 B) buffers, allocated once. `seal` reuses the Vec: +pass capacity ≥ `ciphertext_len(len)` to avoid a realloc+copy (SQL artifacts: allocate that up front; +call seal/open in `spawn_blocking` for large payloads). +Key-material residuals (accepted): `aes`/`aes-gcm` `zeroize` wipe the AES key +schedule, but **not** the GHASH key H (polyval 0.6.2's autodetect backend holds it in a +`ManuallyDrop` union with no `Drop`), nor the `hkdf` 0.12 PRK or `hmac` 0.12 state (keyed by +DEK-derived material / NK). Copies may remain in freed memory inside the TEE. + +Range flow for S3: GET `bytes=0-63` → `Header::parse` → provider.key → `header.map_range(spec)` +(None → 416 with `header.plaintext_len()` as total) → GET `bytes=ciphertext_start-ciphertext_end` +→ `DecryptReader::range`; `Content-Range` from `plaintext_range()`/`total_size()`. + +> **Note for storage callers — header fields are hints until a segment verifies.** +> `Header::parse` checks structure only. The header is authenticated only as AAD of +> each segment, so `plaintext_len()` / `dek_id()` from a header alone can be forged by +> the storage provider (a forged header parses and `map_range` answers `None`). Metadata only, never plaintext, but: +> - HEAD / `Content-Length` / ETag-only paths and 416 totals must not present a +> header-only size as verified. Every segment's AAD includes the header, so decrypting +> **any one** segment (e.g. the first, or the one a range needs anyway) authenticates +> `plaintext_len`/`dek_id`; otherwise record "unverified size" as an accepted risk in +> the storage design and treat it as a hint (e.g. quota math). +> - A range response's `Content-Range` total becomes trustworthy once the first fetched +> segment verifies; send headers after that, not before. +> - `UnknownKey` for a header `dek_id` is an integrity failure of the blob, not 404. +> - **`EnvelopeError::Unsupported` is unauthenticated.** It comes from header bytes 3–4 (or +> the encoding column), which anyone with storage/DB access can flip. KV storage, SQL artifacts and backfill must +> **fail closed** on it: never treat it as legacy plaintext, never re-seal, rewrite, skip +> or garbage-collect the blob because of it (backfill and verify passes included); surface +> it as a hard error (503/500 with "unsupported at-rest format", logged), exactly like an +> integrity failure except for the operator message. +> - `EncryptReader` now seals the final segment only after the source reports EOF, so a +> consumer that stops at `ciphertext_len` cannot store a valid prefix of over-long input; +> still pass `content_length = ciphertext_len(plaintext_len)` and treat any read error +> as a failed PUT. + +## 4. Format (normative; KAT vectors in `at_rest/testdata/vectors.json`, generated by the independent Python reference `at_rest/testdata/gen_vectors.py`) + +Header, 64 bytes, all integers big-endian: +``` +0 "TCE1" 4 +4 version = 1 1 +5 alg = 1 1 (AES-256-GCM, HKDF-SHA256, 64 KiB segments) +6 flags = 0 1 (must be 0) +7 reserved = 0 1 +8 dek_id 4 u32 +12 plaintext_len 8 u64 +20 salt 32 random per blob +52 seg_log2 = 16 1 +53 zero pad 11 (must be 0) +``` +Context (identity) bytes, every variable field `u32 BE len ‖ bytes`: +``` +ctx = lp("tinycloud/at-rest/v1") ‖ lp(purpose) ‖ lp(field_1) ‖ … ‖ lp(field_n) +purpose ∈ {"kv-block","sql-checkpoint","sql-wal"} +kv-block: space, content_hash (multihash bytes) +sql-checkpoint: service, space, name, checkpoint_content_hash, u64 BE checkpoint_size_bytes +sql-wal: service, space, name, i64 BE revision, checkpoint_content_hash, delta_content_hash +``` +Keys and nonces: +``` +K_obj = HKDF-SHA256(ikm = DEK, salt = header.salt, info = SHA-256(ctx), L = 32) +nonce_i = u32 BE i ‖ u8 last(i) ‖ 7 × 0x00 last(i) = 1 iff i == n-1 +AAD_i = ctx ‖ header(64) +n = max(1, ceil(plaintext_len / 65536)) empty plaintext = one empty segment +segment_i = AES-256-GCM(K_obj, nonce_i, AAD_i, pt[i·64Ki .. min((i+1)·64Ki, len)]) = ct ‖ tag16 +ciphertext_len = 64 + plaintext_len + 16·n +``` + +**Versioning and the TCE2 rule (normative).** +- Bytes 0..5 identify the generation: `"TCE" ‖ ASCII digit g ‖ byte g` (g = 1..9). Both the + magic digit (byte 3) and the version byte (byte 4) change together; a mismatch is a + malformed header. TCE1 = `"TCE1" ‖ 0x01`. A TCE2 = `"TCE2" ‖ 0x02` and may change anything + after byte 5 (header length, layout, algorithms). +- Readers call `format_version` on the first 5 bytes **before** generation-specific parsing + and dispatch on g. A generation the build doesn't know → `EnvelopeError::Unsupported{g}` + (operator message: "upgrade the node"), never `Header`/`Integrity`. Not-TCE bytes → `Header`. +- SQL: `payload_encoding`/`delta_encoding` value g (1..=9, the same range the header can + express) means TCE generation g; NULL = legacy plaintext. A value the build doesn't read → + `Unsupported{g}`; 0, negative or >9 → `Header` (invalid). +- KV names stay `v1/…`: the name prefix versions the **naming** scheme (NK/HMAC layout), not the + blob format; a TCE2 blob can live under a `v1/` name, and readers dispatch on the bytes' + first 5 bytes once the name says "sealed". +- **Writers stay on TCE1 until every binary in the rollback window can read TCE2.** That is a + config/mode gate set by the operator after a release that reads TCE2 has been deployed + everywhere and rollback below it is ruled out — not something the column or name decides. + +## 5. Legacy vs sealed: explicit, never sniffed + +```rust +/// SQL: `database_artifact.payload_encoding` / `delta_encoding` (SMALLINT, NULL = legacy). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadEncoding { LegacyPlaintext, Tce1 } +impl PayloadEncoding { + pub fn from_column(v: Option) -> Result; // None→Legacy, 1→Tce1, 2..=9→Unsupported{g}, else Header + pub fn to_column(self) -> Option; +} + +/// KV: by S3 name namespace. Legacy names stay `{space}/{b64url(digest)}`. +pub fn space_tag(nk: &ObjectNameKey, space: &str) -> String; // b64url(HMAC(NK, lp("space")‖lp(space))[..16]) +pub fn sealed_object_name(nk: &ObjectNameKey, space: &str, hash: &Hash) -> String; + // "v1/{space_tag}/{b64url(HMAC(NK, lp("obj")‖lp(space)‖lp(hash)))}" +``` +`open` on bytes that lack the `TCE1` header returns `EnvelopeError::Header`; it never falls +back to plaintext. Whether a blob is legacy is decided by the caller from the name or the +column, before any bytes are read. + +## 6. Test helper +`at_rest::FixedKeyProvider` — in-memory `DataKeyProvider` with fixed DEKs per +(space, dek_id) and a `locked()` constructor. No persistence. **Compiled only under +`cfg(test)` in tinycloud-core or with the `test-util` feature**: downstream crates +add `tinycloud-core = { …, features = ["test-util"] }` to `[dev-dependencies]` only. Debug is +redacted. + +## 7. KV block storage decisions +- **Decided deviation from TC-887 plan §2 (dedup):** in `encrypt`/`strict`, `contains` checks only the + sealed name the write would use. An existing legacy plaintext copy does **not** suppress the + sealed write, so a new put never leaves plaintext as the only copy; the backfill handles only blobs + that existed before. +- **Accepted residual (header trust):** `Content-Length`/`Content-Range`/416 totals come from the + unverified header. A forged header can only produce a wrong total or a body that aborts with + no unauthenticated bytes (test `storage::sealed::tests::forged_header_length_aborts_without_extra_bytes`), + never wrong bytes. +- **Quota:** plaintext bytes. Sealed objects are scanned by `space_tag` and joined to a space when + the object-name key is unlocked (lazily, per lookup), so the first quota check after an unlock + counts legacy + sealed. While locked, a space reports legacy bytes only (writes are 503 then). + A sealed object with an invalid TCE1 length counts 0 bytes; the warning is rate-limited and + carries only the length. +- **Errors:** at-rest causes map explicitly to 503 (keys unavailable / key store) or 500 + (integrity, unsupported, legacy refused in strict) on `/invoke` and the KV GET routes. The + general "store error → 401" catch-all is tracked separately (TC-1347). +- **Reserved names:** The key-continuity key mirror lives under `_tc/` in the block bucket (`datadir/at-rest` on + FS). The block boot scan skips it. diff --git a/tinycloud-core/src/at_rest/envelope.rs b/tinycloud-core/src/at_rest/envelope.rs index c1f401c6..ecd33933 100644 --- a/tinycloud-core/src/at_rest/envelope.rs +++ b/tinycloud-core/src/at_rest/envelope.rs @@ -1,5 +1,5 @@ //! TCE1: a segmented AES-256-GCM envelope (TC-887 plan §1.2–1.3; normative -//! format in `INTERFACES.md` §4). +//! format in `docs/at-rest/INTERFACES.md` §4). //! //! ```text //! header(64) ‖ seg_0 ‖ … ‖ seg_{n-1} seg_i = AES-GCM(ct_i) ‖ tag16 @@ -28,7 +28,8 @@ pub const SEGMENT_LEN: usize = 1 << 16; pub const TAG_LEN: usize = 16; /// Every TCE generation starts `"TCE" ‖ ASCII digit g ‖ version byte g`; this -/// build reads and writes generation 1 only. See `INTERFACES.md` §4 (TCE2 rule). +/// build reads and writes generation 1 only. See `docs/at-rest/INTERFACES.md` +/// §4 (TCE2 rule). const FAMILY: &[u8; 3] = b"TCE"; const MAGIC: &[u8; 4] = b"TCE1"; const VERSION: u8 = 1; diff --git a/tinycloud-core/src/at_rest/names.rs b/tinycloud-core/src/at_rest/names.rs index dd591204..1868c3fd 100644 --- a/tinycloud-core/src/at_rest/names.rs +++ b/tinycloud-core/src/at_rest/names.rs @@ -9,9 +9,10 @@ use hmac::{Hmac, Mac}; use sha2::Sha256; /// `database_artifact.payload_encoding` / `delta_encoding` (SMALLINT). -/// `NULL` is legacy plaintext; `g` in 1..=255 is TCE generation `g`, of which -/// this build reads only 1 (later ones are [`EnvelopeError::Unsupported`]). -/// Zero, negative and larger values are refused as invalid. +/// `NULL` is legacy plaintext; `g` in 1..=9 is TCE generation `g` (the range +/// the header's ASCII digit can express), of which this build reads only 1 +/// (later ones are [`EnvelopeError::Unsupported`]). Zero, negative and larger +/// values are refused as invalid. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PayloadEncoding { LegacyPlaintext, @@ -23,7 +24,7 @@ impl PayloadEncoding { match value { None => Ok(Self::LegacyPlaintext), Some(1) => Ok(Self::Tce1), - Some(v @ 2..=255) => Err(EnvelopeError::Unsupported { version: v as u8 }), + Some(v @ 2..=9) => Err(EnvelopeError::Unsupported { version: v as u8 }), Some(_) => Err(EnvelopeError::Header("invalid payload encoding")), } } diff --git a/tinycloud-core/src/at_rest/provider.rs b/tinycloud-core/src/at_rest/provider.rs index 47f34a2f..a5273614 100644 --- a/tinycloud-core/src/at_rest/provider.rs +++ b/tinycloud-core/src/at_rest/provider.rs @@ -1,5 +1,5 @@ //! The boundary between key continuity (who holds DEKs) and the envelope -//! (what a DEK encrypts). See `INTERFACES.md` §2 on TC-887. +//! (what a DEK encrypts). See `docs/at-rest/INTERFACES.md` §2. use super::key::{DataKey, DekId, ObjectNameKey}; use async_trait::async_trait; diff --git a/tinycloud-core/src/at_rest/tests.rs b/tinycloud-core/src/at_rest/tests.rs index 424ccff7..c63079d3 100644 --- a/tinycloud-core/src/at_rest/tests.rs +++ b/tinycloud-core/src/at_rest/tests.rs @@ -766,7 +766,7 @@ fn payload_encoding_column_is_explicit() { PayloadEncoding::from_column(Some(1)).unwrap(), PayloadEncoding::Tce1 ); - for v in [2i16, 3, 255] { + for v in [2i16, 3, 9] { assert!( matches!( PayloadEncoding::from_column(Some(v)), @@ -775,7 +775,7 @@ fn payload_encoding_column_is_explicit() { "{v}" ); } - for v in [0, -1, 256, i16::MAX, i16::MIN] { + for v in [0, -1, 10, 255, 256, i16::MAX, i16::MIN] { assert!( matches!( PayloadEncoding::from_column(Some(v)), From c2a4435e3bfe60ba00ba43f17b153641432c6bf5 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 08:36:32 +0000 Subject: [PATCH 3/4] test(TC-887): at-rest RED size floor and order-free KV pairing Review notes on #324 (N2, N3): - Every blob that must hold a plaintext (each KV put's objects in total, every SQL payload and delta) must be at least that large: TCE1 never shrinks data, so a smaller blob is compressed, not sealed. Catches xz/bz2-style compression of a salted WAL delta that the shared-run and compressibility checks miss. - KV cross-space comparison requires equal object counts per put and compares every pair, instead of zipping objects sorted by name. - The offline selftest also proves a salted deflate-9 blob is flagged by the floor and sealed-like blobs pass it. --- .../tests/at_rest_harness.rs | 89 ++++++++++++++++--- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/tinycloud-node-server/tests/at_rest_harness.rs b/tinycloud-node-server/tests/at_rest_harness.rs index 27eb7b86..6b63e106 100644 --- a/tinycloud-node-server/tests/at_rest_harness.rs +++ b/tinycloud-node-server/tests/at_rest_harness.rs @@ -20,8 +20,9 @@ //! (SQL). Besides the marker, the space id/address and SQLite magic, they //! reject the plaintext fingerprint in object names and any deterministic, //! non-encrypting transform (encoding, XOR, compression): the two spaces' -//! stored bytes must share no [`MAX_SHARED_RUN`]-byte run, and stored bytes -//! must not compress below [`MIN_INCOMPRESSIBLE_RATIO`]. +//! stored bytes must share no [`MAX_SHARED_RUN`]-byte run, stored bytes +//! must not compress below [`MIN_INCOMPRESSIBLE_RATIO`], and a blob must be +//! at least as large as the plaintext it holds (TCE1 never shrinks data). use std::collections::{BTreeSet, HashMap}; use std::io::Write; @@ -461,6 +462,20 @@ fn check_incompressible(what: &str, bytes: &[u8], findings: &mut Vec) -> Ok(()) } +/// Finding if `stored` bytes are fewer than the `plaintext` bytes they must +/// hold. TCE1 adds a header and per-segment tags and never shrinks data, so a +/// smaller blob is compressed (or truncated), not sealed. This catches strong +/// compressors (xz, bz2) whose output neither compresses further nor repeats +/// across writes once a per-write salt (e.g. SQLite's WAL salts) is mixed in. +fn check_size_floor(what: &str, stored: usize, plaintext: usize, findings: &mut Vec) { + if stored < plaintext { + findings.push(format!( + "{what} is {stored} bytes, smaller than the {plaintext}-byte plaintext it must hold \ + (sealing never shrinks data: compressed, not sealed)" + )); + } +} + /// A text encoding of the plaintext's blake3 fingerprint. struct Fingerprint { label: String, @@ -611,6 +626,8 @@ struct SqlSpace { /// The same SQL statements run in two fresh spaces. struct SqlRun { markers: [String; 2], + /// Bytes of the smaller 64 KiB value; every payload and delta holds one. + big_len: usize, spaces: Vec, } @@ -620,6 +637,7 @@ async fn sql_run(harness: &Harness) -> Result { repeated(&markers[0], BIG_VALUE_LEN), repeated(&markers[1], BIG_VALUE_LEN), ]; + let big_len = big[0].len().min(big[1].len()); let insert = "INSERT INTO probe (secret) VALUES (?)"; let writes = [ // No checkpoint exists yet: the node saves a full checkpoint into @@ -665,7 +683,11 @@ async fn sql_run(harness: &Harness) -> Result { sql_reached_postgres(&space)?; spaces.push(space); } - Ok(SqlRun { markers, spaces }) + Ok(SqlRun { + markers, + big_len, + spaces, + }) } /// The writes reached PostgreSQL as a checkpoint and a WAL delta. @@ -755,6 +777,12 @@ async fn red_kv_blocks_are_sealed_in_minio() -> Result<()> { let space_id = space.account.space.to_string(); let address = space.account.address_hex(); for ((label, value), objects) in run.probes.iter().zip(&space.objects) { + check_size_floor( + &format!("KV {label} put for {space_id}: its new object(s) in total"), + objects.iter().map(|object| object.body.len()).sum(), + value.len(), + &mut findings, + ); let fingerprints = fingerprints(value.as_bytes()); for StoredObject { key, body } in objects { if key.contains(&space_id) { @@ -802,16 +830,28 @@ async fn red_kv_blocks_are_sealed_in_minio() -> Result<()> { } } } + // Compare every object of a put in space A with every object of the same + // put in space B: sealed names sort randomly, so no pairing by order. for (index, (label, _)) in run.probes.iter().enumerate() { let a = &run.spaces[0].objects[index]; let b = &run.spaces[1].objects[index]; - for (a, b) in a.iter().zip(b) { - compare_writes( - &format!("KV {label} value ({:?} vs {:?})", a.key, b.key), - &a.body, - &b.body, - &mut findings, - ); + ensure!( + a.len() == b.len(), + "harness: the same KV {label} put created {} object(s) in {} but {} in {}", + a.len(), + run.spaces[0].account.space, + b.len(), + run.spaces[1].account.space + ); + for a in a { + for b in b { + compare_writes( + &format!("KV {label} value ({:?} vs {:?})", a.key, b.key), + &a.body, + &b.body, + &mut findings, + ); + } } } ensure!( @@ -863,6 +903,12 @@ async fn red_sql_artifacts_are_sealed_in_postgres() -> Result<()> { } } check_incompressible(&format!("{id}.{column}"), bytes, &mut findings)?; + check_size_floor( + &format!("{id}.{column}"), + bytes.len(), + run.big_len, + &mut findings, + ); } } for a in &run.spaces[0].rows { @@ -930,8 +976,9 @@ fn sealed_like(len: usize) -> Vec { /// Offline check of the RED predicates themselves (runs in plain `cargo /// test`, no harness; its name stays clear of the `green_`/`red_` filters /// `harness.sh test` uses): deterministic transforms of identical plaintext -/// are flagged even with the marker and magic gone, and two sealed-like -/// blobs of the same plaintext are not. +/// are flagged even with the marker and magic gone, a compressed-only blob is +/// flagged by the size floor, and two sealed-like blobs of the same plaintext +/// pass every predicate. #[test] fn selftest_predicates_flag_deterministic_transforms_not_ciphertext() -> Result<()> { let marker = marker("SELFTEST"); @@ -969,10 +1016,28 @@ fn selftest_predicates_flag_deterministic_transforms_not_ciphertext() -> Result< "{name} behind a random prefix: shared-run check missed it" ); } + // Compressed only, with a per-write salt mixed in (like a WAL delta): + // the size floor still catches it. + let deflate_best = |bytes: &[u8]| -> Result> { + let mut encoder = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::best()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) + }; + let salted = deflate_best(&[random_bytes(32), plaintext.clone()].concat())?; + let mut findings = Vec::new(); + check_size_floor("deflate-9", salted.len(), plaintext.len(), &mut findings); + ensure!( + findings.len() == 1, + "a {}-byte compressed-only blob of {} bytes passed the size floor", + salted.len(), + plaintext.len() + ); for len in [62, BIG_VALUE_LEN] { let (a, b) = (sealed_like(len), sealed_like(len)); let mut findings = Vec::new(); compare_writes("sealed", &a, &b, &mut findings); + check_size_floor("sealed", a.len(), len, &mut findings); if len == BIG_VALUE_LEN { check_incompressible("sealed", &a, &mut findings)?; } From efe1f851bf69e2f076018e5b94096dde66761fe1 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 08:36:32 +0000 Subject: [PATCH 4/4] test(TC-887): harness ESCROW_NONE mapping and per-check expectations Review notes on #324 (N1, N4): - TC887_AT_REST_ESCROW_NONE=true|false maps to S1's boolean TINYCLOUD_AT_REST__ESCROW_NONE (replaces the scalar ESCROW, which S1's deny_unknown_fields section rejects); mode is validated as off|encrypt|strict. State schema v3. - test --expect takes green|red or kv=green|red,sql=green|red. Each check runs alone and is judged on its own; exit 0 only if all match, 3 per mismatch (KV/SQL-specific UNEXPECTEDLY PASSED / still RED), 4 if any check errored. With mode encrypt|strict, test red defaults to kv=green,sql=red (S3 before S4); CI stays --expect red. - README: SIGKILL of up can't be trapped; run down STATE. --- .github/workflows/rust.yml | 4 +- scripts/at-rest-harness/README.md | 47 ++++--- scripts/at-rest-harness/harness.sh | 204 ++++++++++++++++++----------- 3 files changed, 162 insertions(+), 93 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e2ac72d5..8e3f7c19 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -246,8 +246,8 @@ jobs: # database_artifact (checkpoint and WAL delta). The RED checks assert # encryption at rest; until TC-887 S3/S4 land, the step passes only if both # fail with their "TC-887 RED:" findings (`--expect red`), and fails loudly - # if they error or unexpectedly pass. Switch to `--expect green` (with the - # at-rest mode set) when S3/S4 merge. + # if they error or unexpectedly pass. When S3 lands, start the harness with + # the at-rest mode set and expect `kv=green,sql=red`; when S4 lands, `green`. at-rest-harness: name: at-rest harness (native PostgreSQL + MinIO) runs-on: ubuntu-latest diff --git a/scripts/at-rest-harness/README.md b/scripts/at-rest-harness/README.md index 1639d362..25f8c8eb 100644 --- a/scripts/at-rest-harness/README.md +++ b/scripts/at-rest-harness/README.md @@ -36,7 +36,7 @@ in logs. `state.json` (ports, PIDs and start times, binaries, log paths, ```sh STATE=$(scripts/at-rest-harness/harness.sh up) # builds the node (nice -n 10, -j 8); prints the state path last scripts/at-rest-harness/harness.sh test green "$STATE" # writes reach MinIO and PG (passes) -scripts/at-rest-harness/harness.sh test red --expect red "$STATE" # exit 0 only if both RED checks fail as RED +scripts/at-rest-harness/harness.sh test red --expect red "$STATE" # exit 0 only if both RED checks (kv, sql) fail as RED scripts/at-rest-harness/harness.sh down --purge "$STATE" # stop this run's processes, remove its dir ``` @@ -45,8 +45,10 @@ run dir). Nothing defaults to a "latest" run, so concurrent runs on a shared host can't act on each other. - `up --no-build` reuses an already-built binary (or set `TC887_NODE_BIN`). -- If `up` fails, times out or is interrupted, it stops every process it - started. It keeps the run dir with its logs; remove it with `down --purge`. +- If `up` fails, times out (SIGTERM) or is interrupted (SIGINT), it stops + every process it started. It keeps the run dir with its logs; remove it with + `down --purge`. A SIGKILL of `up` can't be trapped: then run `down STATE` + (the state file is written after each start). - `status STATE` prints the state file and `logs STATE` tails the logs. - Overrides: `TC887_SCRATCH_ROOT`, `TC887_RUN_DIR` (must be named `harness-run-*`), `TC887_{PG,MINIO,MINIO_CONSOLE,NODE}_PORT`, `TC887_BIN_DIR`, @@ -55,30 +57,37 @@ host can't act on each other. take one before the harness does. A collision makes `up` fail cleanly; it can't produce a false green, because the checks talk to MinIO and PG directly. -`test` exits 0 only on the expected outcome: +`test` runs each check (`kv`, `sql`) on its own and compares each with its +expectation. `--expect` takes `green` or `red` (both checks) or a per-check +list such as `kv=green,sql=red`; a check left out keeps its default. `test` +exits 0 only if every check matches: | Exit | Meaning | |---|---| -| 0 | as expected (`GREEN as expected` / `RED as expected`) | -| 3 | unexpected result: RED checks passed (`UNEXPECTEDLY PASSED for KV/SQL`), or `--expect green` while still RED | -| 4 | errored: compile failure, harness failure, a precondition (`harness no-op`/`harness gap`) or a node error, i.e. not a RED finding | +| 0 | every check as expected (`KV RED as expected` … / `RED as expected: both …`) | +| 3 | a check didn't match: `KV red check UNEXPECTEDLY PASSED under expect kv=red`, or `KV red check still RED … but expected kv=green` (per check) | +| 4 | a check errored: compile failure, harness failure, a precondition (`harness no-op`/`harness gap`) or a node error, i.e. not a RED finding (per check; wins over 3) | -`--expect` defaults to `green` for `test green`. For `test red` it defaults to -`red`, or to `green` if the run was started with `TC887_AT_REST_MODE=encrypt` -or `strict`. +Defaults: `test green` expects `kv=green,sql=green`. `test red` expects +`kv=red,sql=red`, or `kv=green,sql=red` if the run was started with +`TC887_AT_REST_MODE=encrypt` or `strict`, because S3 seals KV before S4 seals +SQL. When S4 lands, the SQL default flips to green (`default_expect` in +`harness.sh`). CI runs `test red --expect red` until S3 lands. ## At-rest mode -The config names belong to the S1 key-continuity slice and may change. They -are defined in one marked block at the top of `harness.sh`: +The names are S1's (key continuity). They are defined in one marked block at +the top of `harness.sh`: | Harness env (at `up`) | Node env written to `node.env` | |---|---| | `TC887_AT_REST_MODE` (`off`/`encrypt`/`strict`) | `TINYCLOUD_AT_REST__MODE` | -| `TC887_AT_REST_ESCROW` (e.g. `none`, the Static self-host acknowledgement, plan §5) | `TINYCLOUD_AT_REST__ESCROW` | +| `TC887_AT_REST_ESCROW_NONE` (`true`/`false`: the Static self-host acknowledgement that no escrow exists, plan §5) | `TINYCLOUD_AT_REST__ESCROW_NONE` | If either is unset, the harness sets nothing and the node runs its default -(today: no encryption at rest). +(today: no encryption at rest). Other values are refused before anything +starts. The simplest encrypting node is the harness's Static root plus +`TC887_AT_REST_MODE=encrypt TC887_AT_REST_ESCROW_NONE=true`. ## Checks @@ -98,17 +107,21 @@ then insert (a WAL delta). |---|---|---| | `green_kv_put_reaches_minio` | every put round-trips through the node and adds a non-empty object to the bucket | pass | | `green_sql_write_reaches_postgres` | the writes round-trip, and `database_artifact` has a `payload` and a non-NULL `delta_payload` | pass | -| `red_kv_blocks_are_sealed_in_minio` | object keys contain no space id, wallet address or blake3 fingerprint of the value (b64url padded/unpadded, base64 padded/unpadded, hex; raw digest and multihash); raw bytes hold no marker; the 64 KiB object doesn't compress (zlib ratio > 0.95); the two spaces' objects for the same value share no 32-byte run | **fail** | -| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker or compresses (ratio > 0.95); the two spaces' payloads and deltas share no 32-byte run | **fail** | +| `red_kv_blocks_are_sealed_in_minio` | object keys contain no space id, wallet address or blake3 fingerprint of the value (b64url padded/unpadded, base64 padded/unpadded, hex; raw digest and multihash); raw bytes hold no marker; each put's objects together are at least as large as the value; the 64 KiB object doesn't compress (zlib ratio > 0.95); every object of a put in one space shares no 32-byte run with every object of the same put in the other (equal object counts required) | **fail** | +| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker, compresses (ratio > 0.95) or is smaller than the 64 KiB value it holds; the two spaces' payloads and deltas share no 32-byte run | **fail** | The RED checks re-run the green preconditions first, so a RED result is never a harness no-op. The 32-byte limit: two sealed blobs of the same plaintext share only fixed TCE1 header fields (≤ 20 contiguous bytes), while any deterministic transform (encoding, XOR, compression, an unsalted cipher) keeps runs as long as the data. The reasoning is in the test's `MAX_SHARED_RUN` doc. +The size floor holds because TCE1 never shrinks data. It catches strong +compressors (xz, bz2) whose output neither compresses further nor repeats +across writes once a per-write salt (SQLite's WAL salts) is mixed in. `selftest_predicates_flag_deterministic_transforms_not_ciphertext` runs in plain `cargo test` and checks the predicates offline: base64, XOR and zlib are -flagged, and sealed-like random blobs are not. +flagged, a salted deflate-9 blob is flagged by the size floor, and sealed-like +random blobs pass. ## Not covered diff --git a/scripts/at-rest-harness/harness.sh b/scripts/at-rest-harness/harness.sh index 0a51e8a9..d46ccc4a 100755 --- a/scripts/at-rest-harness/harness.sh +++ b/scripts/at-rest-harness/harness.sh @@ -6,8 +6,9 @@ # # harness.sh bins fetch + sha256-verify the pinned MinIO/mc binaries # harness.sh up [--no-build] start PG, MinIO, the node; prints the STATE path last -# harness.sh test green|red [--expect green|red] STATE -# run the checks; exit 0 only on the expected outcome +# harness.sh test green|red [--expect EXPECT] STATE +# run the checks; exit 0 only if each check (kv, sql) +# matches EXPECT: green|red (both) or kv=green|red,sql=green|red # harness.sh status STATE print the state file # harness.sh logs STATE tail the harness logs (no secrets are written to them) # harness.sh down [--purge] STATE stop the recorded processes; --purge removes the run dir @@ -19,7 +20,7 @@ set -Eeuo pipefail ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) SCRATCH_ROOT=${TC887_SCRATCH_ROOT:-$HOME/scratch/tc887} BIN_DIR=${TC887_BIN_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/tc887-harness/bin} -STATE_SCHEMA=tc887-at-rest-harness/v2 +STATE_SCHEMA=tc887-at-rest-harness/v3 # Pinned MinIO community releases (linux-amd64). dl.min.io no longer serves # community binaries (HTTP 410), so these come from the archived GitHub @@ -32,16 +33,20 @@ MINIO_URL=https://github.com/minio/minio/releases/download/$MINIO_RELEASE/minio. MC_URL=https://github.com/minio/mc/releases/download/$MC_RELEASE/mc.linux-amd64.$MC_RELEASE # --------------------------------------------------------------------------- -# TC-887 at-rest node config. These two names belong to the S1 key-continuity -# slice and may change; this block is the only place the harness defines them -# (README "At-rest mode" documents the same mapping). -# TC887_AT_REST_MODE -> TINYCLOUD_AT_REST__MODE (off|encrypt|strict) -# TC887_AT_REST_ESCROW -> TINYCLOUD_AT_REST__ESCROW (Static self-host -# `none` acknowledgement, plan §5) +# TC-887 at-rest node config, as defined by the S1 key-continuity slice. This +# block is the only place the harness names them (README "At-rest mode" +# documents the same mapping). +# TC887_AT_REST_MODE -> TINYCLOUD_AT_REST__MODE (off|encrypt|strict) +# TC887_AT_REST_ESCROW_NONE -> TINYCLOUD_AT_REST__ESCROW_NONE (true|false: the Static +# self-host acknowledgement that no escrow exists, plan §5) # Unset: the harness sets nothing and the node runs its default (today: no -# encryption at rest). With mode encrypt/strict, `test red` expects green. +# encryption at rest). Simplest encrypting node: Static root (always, here) + +# TC887_AT_REST_MODE=encrypt + TC887_AT_REST_ESCROW_NONE=true. With mode +# encrypt/strict, `test red` defaults to expecting kv=green,sql=red: S3 seals +# KV before S4 seals SQL. When S4 lands, flip the SQL default to green +# (default_expect below). AT_REST_MODE_VAR=TINYCLOUD_AT_REST__MODE -AT_REST_ESCROW_VAR=TINYCLOUD_AT_REST__ESCROW +AT_REST_ESCROW_NONE_VAR=TINYCLOUD_AT_REST__ESCROW_NONE # --------------------------------------------------------------------------- BUCKET=tinycloud-blocks @@ -229,7 +234,8 @@ up_abort() { exit "$status" } -valid_setting() { [[ "$1" =~ ^[A-Za-z0-9_.:-]+$ ]]; } +valid_mode() { [[ "$1" =~ ^(off|encrypt|strict)$ ]]; } +valid_bool() { [[ "$1" =~ ^(true|false)$ ]]; } cmd_up() { local build=1 @@ -244,9 +250,11 @@ cmd_up() { trap 'exit 143' TERM command -v python3 >/dev/null || die "python3 is required" command -v curl >/dev/null || die "curl is required" - local at_rest_mode=${TC887_AT_REST_MODE:-} at_rest_escrow=${TC887_AT_REST_ESCROW:-} - [[ -z "$at_rest_mode" ]] || valid_setting "$at_rest_mode" || die "invalid TC887_AT_REST_MODE" - [[ -z "$at_rest_escrow" ]] || valid_setting "$at_rest_escrow" || die "invalid TC887_AT_REST_ESCROW" + local at_rest_mode=${TC887_AT_REST_MODE:-} at_rest_escrow_none=${TC887_AT_REST_ESCROW_NONE:-} + [[ -z "$at_rest_mode" ]] || valid_mode "$at_rest_mode" \ + || die "TC887_AT_REST_MODE must be off, encrypt or strict" + [[ -z "$at_rest_escrow_none" ]] || valid_bool "$at_rest_escrow_none" \ + || die "TC887_AT_REST_ESCROW_NONE must be true or false" cmd_bins local pg_bin @@ -287,7 +295,7 @@ cmd_up() { # Recorded by every write_state call, including a failed up's. export STATE_PG_BIN=$pg_bin STATE_NODE_BIN=$node_bin STATE_PG_PORT=$pg_port \ STATE_MINIO_PORT=$minio_port STATE_CONSOLE_PORT=$console_port STATE_NODE_PORT=$node_port \ - STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW=$at_rest_escrow + STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW_NONE=$at_rest_escrow_none # Per-run secrets: generated here, kept in 0600 files, never on argv. random_hex | secret_file "$run/secrets/pg-password" @@ -345,9 +353,9 @@ EOF printf '%s=%s\n' "$AT_REST_MODE_VAR" "$at_rest_mode" >>"$run/node.env" log "at-rest: $AT_REST_MODE_VAR=$at_rest_mode" fi - if [[ -n "$at_rest_escrow" ]]; then - printf '%s=%s\n' "$AT_REST_ESCROW_VAR" "$at_rest_escrow" >>"$run/node.env" - log "at-rest: $AT_REST_ESCROW_VAR=$at_rest_escrow" + if [[ -n "$at_rest_escrow_none" ]]; then + printf '%s=%s\n' "$AT_REST_ESCROW_NONE_VAR" "$at_rest_escrow_none" >>"$run/node.env" + log "at-rest: $AT_REST_ESCROW_NONE_VAR=$at_rest_escrow_none" fi printf '[global]\nlog_level = "normal"\n' >"$run/tinycloud.toml" write_state "$run" starting @@ -413,7 +421,7 @@ write_state() { NODE_PID=$UP_NODE_PID NODE_START=$UP_NODE_START \ BIN_DIR=$BIN_DIR MINIO_RELEASE=$MINIO_RELEASE MC_RELEASE=$MC_RELEASE \ BUCKET=$BUCKET REGION=$REGION PG_USER=$PG_USER PG_DB=$PG_DB \ - AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_VAR=$AT_REST_ESCROW_VAR python3 - <<'PY' + AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_NONE_VAR=$AT_REST_ESCROW_NONE_VAR python3 - <<'PY' import json, os, datetime e = os.environ run = e["RUN"] @@ -433,9 +441,9 @@ state = { "env_file": os.path.join(run, "harness.env"), "at_rest": { "mode": text("STATE_AT_REST_MODE"), - "escrow": text("STATE_AT_REST_ESCROW"), + "escrow_none": {"true": True, "false": False}.get(e.get("STATE_AT_REST_ESCROW_NONE", "")), "mode_var": e["AT_REST_MODE_VAR"], - "escrow_var": e["AT_REST_ESCROW_VAR"], + "escrow_none_var": e["AT_REST_ESCROW_NONE_VAR"], }, "node": { "pid": num("NODE_PID"), @@ -554,74 +562,122 @@ annotate() { log "$*" } +# The harness test for each check, per kind, and the finding its RED test +# prints when the stored bytes are plaintext. +test_name() { + case $1-$2 in + green-kv) echo green_kv_put_reaches_minio ;; + green-sql) echo green_sql_write_reaches_postgres ;; + red-kv) echo red_kv_blocks_are_sealed_in_minio ;; + red-sql) echo red_sql_artifacts_are_sealed_in_postgres ;; + esac +} +red_finding() { + case $1 in + kv) echo 'TC-887 RED: KV data is not sealed at rest' ;; + sql) echo 'TC-887 RED: SQL artifacts are not sealed at rest' ;; + esac +} + +# Default expectation of CHECK for KIND on the harness at STATE. +default_expect() { + local kind=$1 check=$2 state=$3 + [[ "$kind" == green ]] && { echo green; return; } + case $(state_get "$state" at_rest.mode) in + # S3 seals KV before S4 seals SQL; flip sql to green when S4 lands. + encrypt|strict) [[ "$check" == kv ]] && echo green || echo red ;; + *) echo red ;; + esac +} + +# Run one check alone; sets CHECK_OUTCOME (green|red|errored) and CHECK_DETAIL. +run_check() { + local kind=$1 check=$2 state=$3 name out status=0 passed failed + name=$(test_name "$kind" "$check") + out=$(mktemp) + (cd "$ROOT" && TC887_HARNESS_STATE=$state nice -n 10 cargo test -p tinycloud-node \ + -j "${TC887_CARGO_JOBS:-8}" --test at_rest_harness -- --ignored --nocapture --exact "$name") \ + 2>&1 | tee "$out" || status=$? + read -r passed failed < <(grep -Eo 'test result: [A-Za-z]+\. [0-9]+ passed; [0-9]+ failed' "$out" \ + | awk '{p += $4; f += $6} END {print p + 0, f + 0}') + CHECK_DETAIL="cargo exit $status" + # libtest exits 0 when a filter matches nothing; never let that pass (TC-381). + if ((passed + failed != 1)); then + CHECK_OUTCOME=errored + CHECK_DETAIL="expected 1 test ($name) to run, $((passed + failed)) ran; $CHECK_DETAIL (compile or harness failure)" + elif ((passed == 1 && status == 0)); then + CHECK_OUTCOME=green + elif [[ "$kind" == red ]] && grep -qF "$(red_finding "$check")" "$out"; then + CHECK_OUTCOME=red + else + CHECK_OUTCOME=errored + CHECK_DETAIL="$name failed without a TC-887 RED finding (harness, precondition or node error; see output above)" + fi + rm -f "$out" +} + cmd_test() { local kind="" expect="" target="" while (($#)); do case $1 in green|red) [[ -z "$kind" ]] || die "test: one kind only"; kind=$1 ;; - --expect) (($# >= 2)) || die "--expect needs green or red"; shift; expect=$1 ;; + --expect) (($# >= 2)) || die "--expect needs a value"; shift; expect=$1 ;; --expect=*) expect=${1#--expect=} ;; -*) die "test: unknown option $1" ;; *) [[ -z "$target" ]] || die "test: one STATE only"; target=$1 ;; esac shift done - [[ -n "$kind" ]] || die "usage: harness.sh test green|red [--expect green|red] STATE" + [[ -n "$kind" ]] || die "usage: harness.sh test green|red [--expect EXPECT] STATE" local state state=$(resolve_state "$target") [[ "$(state_get "$state" status)" == up ]] || die "harness at $state is not up" - if [[ -z "$expect" ]]; then - expect=green - if [[ "$kind" == red ]]; then - case $(state_get "$state" at_rest.mode) in - encrypt|strict) expect=green ;; - *) expect=red ;; - esac - fi - fi - [[ "$expect" == green || "$expect" == red ]] || die "--expect must be green or red" - local out status=0 - out=$(mktemp) - (cd "$ROOT" && TC887_HARNESS_STATE=$state nice -n 10 cargo test -p tinycloud-node \ - -j "${TC887_CARGO_JOBS:-8}" --test at_rest_harness -- --ignored --nocapture --test-threads=1 "${kind}_") \ - 2>&1 | tee "$out" || status=$? - local passed failed kv_red=0 sql_red=0 - read -r passed failed < <(grep -Eo 'test result: [A-Za-z]+\. [0-9]+ passed; [0-9]+ failed' "$out" \ - | awk '{p += $4; f += $6} END {print p + 0, f + 0}') - if [[ "$kind" == red ]]; then - grep -q 'TC-887 RED: KV data is not sealed at rest' "$out" && kv_red=1 - grep -q 'TC-887 RED: SQL artifacts are not sealed at rest' "$out" && sql_red=1 - fi - rm -f "$out" - local reds=$((kv_red + sql_red)) ran=$((passed + failed)) - - # libtest exits 0 when a filter matches nothing; never let that pass (TC-381). - if ((ran != 2)); then - annotate error "$kind checks ERRORED: expected 2 to run, $ran ran (cargo exit $status: compile or harness failure)" - return 4 - fi - if ((failed != reds)); then - annotate error "$kind checks ERRORED: $failed failed but only $reds with a TC-887 RED finding (harness, precondition or node error; see output above)" - return 4 - fi - if [[ "$expect" == green ]]; then - if ((status == 0 && failed == 0)); then - annotate notice "$kind checks GREEN as expected ($passed passed)" - return 0 + local -A want=([kv]=$(default_expect "$kind" kv "$state") [sql]=$(default_expect "$kind" sql "$state")) + local item key value items=() + case $expect in + "") ;; + green|red) want[kv]=$expect want[sql]=$expect ;; + *) + IFS=, read -ra items <<<"$expect" + for item in "${items[@]}"; do + key=${item%%=*} value=${item#*=} + [[ "$item" == *=* && ( "$key" == kv || "$key" == sql ) && ( "$value" == green || "$value" == red ) ]] \ + || die "--expect must be green, red, or kv=green|red,sql=green|red (got '$item')" + want[$key]=$value + done + ;; + esac + + local check rc=0 CHECK_OUTCOME CHECK_DETAIL label + for check in kv sql; do + run_check "$kind" "$check" "$state" + label=$(tr '[:lower:]' '[:upper:]' <<<"$check") + if [[ "$CHECK_OUTCOME" == errored ]]; then + annotate error "$label $kind check ERRORED: $CHECK_DETAIL" + rc=4 + elif [[ "$CHECK_OUTCOME" == "${want[$check]}" ]]; then + if [[ "$CHECK_OUTCOME" == red ]]; then + annotate notice "$label RED as expected: $label data is plaintext at rest" + else + annotate notice "$label $kind check GREEN as expected" + fi + elif [[ "$CHECK_OUTCOME" == green ]]; then + annotate error "$label $kind check UNEXPECTEDLY PASSED under expect $check=red: the at-rest property holds for $label; expect $check=green and update CI" + ((rc == 4)) || rc=3 + else + annotate error "$label $kind check still RED ($label data is plaintext at rest) but expected $check=green" + ((rc == 4)) || rc=3 + fi + done + if ((rc == 0)); then + if [[ "$kind" == red && "${want[kv]}" == red && "${want[sql]}" == red ]]; then + annotate notice "RED as expected: both TC-887 RED checks report plaintext at rest (KV and SQL)" + else + annotate notice "$kind checks as expected (kv=${want[kv]}, sql=${want[sql]})" fi - annotate error "$kind checks still RED ($reds of 2 report plaintext at rest) but expected green" - return 3 - fi - if ((status == 101 && reds == 2)); then - annotate notice "RED as expected: both TC-887 RED checks report plaintext at rest (KV and SQL)" - return 0 fi - local flipped=() - ((kv_red)) || flipped+=(KV) - ((sql_red)) || flipped+=(SQL) - annotate error "$kind checks UNEXPECTEDLY PASSED under --expect red (no TC-887 RED finding for ${flipped[*]}): the at-rest property holds there; run with --expect green (or set TC887_AT_REST_MODE) and update CI" - return 3 + return "$rc" } cmd_status() { @@ -647,5 +703,5 @@ case ${1:-} in test) shift; cmd_test "$@" ;; status) shift; cmd_status "$@" ;; logs) shift; cmd_logs "$@" ;; - *) die "usage: harness.sh bins | up [--no-build] | test green|red [--expect green|red] STATE | status STATE | logs STATE | down [--purge] STATE" ;; + *) die "usage: harness.sh bins | up [--no-build] | test green|red [--expect EXPECT] STATE | status STATE | logs STATE | down [--purge] STATE" ;; esac