diff --git a/Cargo.lock b/Cargo.lock index 94879464..4154bf6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10290,6 +10290,7 @@ name = "tinycloud-node" version = "1.20.7" dependencies = [ "aes-gcm", + "age", "anyhow", "async-trait", "aws-config", @@ -10316,6 +10317,7 @@ dependencies = [ "lazy_static", "libc", "libp2p-identity", + "lru", "multer", "native-tls", "opentelemetry", @@ -10354,6 +10356,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tinycloud-core", "tinycloud-sdk-wasm", diff --git a/docs/at-rest-escrow.md b/docs/at-rest-escrow.md index 908477e9..74b7cf4b 100644 --- a/docs/at-rest-escrow.md +++ b/docs/at-rest-escrow.md @@ -1,6 +1,6 @@ # At-rest key escrow (TC-887) -Nothing in the node binary uses these two crates yet. +The node provisions and escrows at-rest keys only when `at_rest` is configured (below). The default config does nothing: no key reads, no rows. - `tinycloud-at-rest-escrow` (a node workspace member) holds the escrow format: - the 2-of-3 split of the at-rest root key (ARK); @@ -124,3 +124,72 @@ Trust policy (`--trust`), from a source independent of the node being checked: "allowed_advisories": [] } ``` + +## Node configuration and endpoints + +```toml +[at_rest] +mode = "off" # off | encrypt | strict +escrow_none = false # self-hosted Static-root nodes only (refused in a TEE) + +[at_rest.escrow] +recipients = [ { holder = "sam", recipient = "age1yubikey1…" }, … ] # exactly 3 +verified_fp = "" # after the ceremony (record-efp) +allow_software = false # X25519 holders; tests and self-host only, refused in a TEE +allow_resplit_before_verify = false # before the ceremony only: corrected holders get a fresh root key +``` + +The same settings as compose env: `TINYCLOUD_AT_REST__MODE`, `TINYCLOUD_AT_REST__ESCROW_NONE`, `TINYCLOUD_AT_REST__ESCROW__RECIPIENTS=[{holder="…",recipient="…"},…]`, `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP`, `TINYCLOUD_AT_REST__ESCROW__ALLOW_SOFTWARE`, `TINYCLOUD_AT_REST__ESCROW__ALLOW_RESPLIT_BEFORE_VERIFY`. + +- **Recipients set, `mode = "off"`:** + - at boot the node creates the ARK (sealed under its root) and the escrow bundle; + - it keeps a copy of the sealed key, the bundle, the ceremony record and each space's wrapped data key in the block store (`_tc/at-rest/` in S3, `{datadir}/at-rest/_tc/at-rest/` on disk). Data-key objects are named by a keyed hash, not the space; + - in this mode a copy that can't be written is logged and the boot goes on (nothing depends on it yet). With `encrypt` or `strict`, every boot-time copy must be written: a block store that refuses the write stops the boot; + - nothing is encrypted. +- **`encrypt` and `strict` are refused at boot in this build.** Block storage has no seal path yet, so the node won't claim to encrypt while it writes plaintext; `/info` never shows `encrypting: true`. Storage turns the capability on when the seal path lands (TC-887 S3). From then on, `encrypt` or `strict` refuses to start unless the recipients equal the bundle's, `verified_fp` equals its EFP, and the stored ceremony record verifies under the key inside the TEE. A Static-root node with no recipients may use `escrow_none = true` instead, which acknowledges that no escrow exists. +- **In a TEE, the escrow recipients and `verified_fp` must be literal values in the measured compose.** With any at-rest config, the node reads its `app-compose.json` from the guest agent (`/Info`, `tcb_info.app_compose`), checks it against `compose_hash`, and refuses to boot unless `TINYCLOUD_AT_REST__ESCROW__RECIPIENTS` and `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP` are set there as literals (no `${…}`, no pass-through) equal to the effective config. Values from the encrypted environment, a file or anywhere else are refused. If the app-compose can't be read or parsed, the node refuses to boot. The `/Info` shape must be confirmed on a live CVM before the escrow ceremony. +- **The escrow bundle is made or changed only at boot, never by a request.** Every endpoint below only reads it. + - **An ARK never gets a second bundle for other holders.** The node refuses to boot (`escrow_holders_changed`) if the configured recipients differ from any existing bundle for its ARK: the active bundle, any stored escrow row, a ceremony record in the block store whose MAC verifies under the ARK, or a bundle object in the block store naming the ARK. This holds after a restore too, when the database lost its escrow rows. The block-store bundle object is unauthenticated, so it can only refuse, never be used. Changing a holder after the ceremony needs an ARK rotation (TC-1345). + - **Fixing a wrong holder before the ceremony:** set `allow_resplit_before_verify = true` together with the corrected recipients. The node then **replaces the root key**: it makes a fresh ARK, splits it for the corrected holders, and retires the old ARK and its bundle. Shares of the old bundle become worthless: any two of them rebuild only the retired key, which never protected anything and is never loaded again (a W-authenticated retirement marker in the block store drops it even from a database restored to before the fix). The node does this only if no fingerprint is pinned, no ceremony record exists, and no data key was ever made, checking both the database and the block store. If any of those fails, it still refuses to boot. Remove the flag after the fix. + - With `verified_fp` set, the node uses only the pinned bundle, restoring it from the block store if the database lost it, and refuses to boot if that bundle is not for the configured recipients and the newest key. +- **If the node's root can no longer unseal the key** (new app id or KMS), it boots **locked**: + - `/info` shows `at_rest.state = "locked"` and encryption is unavailable; + - two holders recover it through the endpoints below, which needs `verified_fp` to be pinned; + - the node then re-seals the key under its new root, with no restart and no data rewrite. + +| Endpoint | Answer | +|---|---| +| `GET /at-rest/status` | `mode`, `state`, `encrypting`, `efp`, `generation`, `verified` | +| `GET /at-rest/escrow` | the active escrow bundle (or, on a locked node, the pinned one) | +| `GET /at-rest/escrow/record` | the ceremony record, only if its MAC verifies under the live key | +| `GET /at-rest/recovery?nonce=N` | `RecoveryInfo`: this boot's recovery recipient, bound with `N` and the EFP into a TDX quote (`mode = "classic"` with no quote outside a TEE) | +| `POST /at-rest/recovery/shares` | body: one share age-sealed to the recovery recipient (≤ 4 KiB). Answers `held` (with `shares_held`), `verified` (all three shares rebuilt the key), or `unlocked` (locked node recovered), with the gate result | + +The share endpoint: +- is rate limited per IP. Until TC-1357, the client IP comes from a proxy header trusted from any peer, so per-client limits are best effort; the per-boot budget below doesn't depend on them; +- checks every share against the bundle's commitments before it takes one of three slots; +- counts only shares that authenticate (decrypt and match a commitment) against the node's per-boot share budget, so junk can't starve genuine holders. Rejected shares are limited per client (5, then one a minute); +- never logs share bytes or ciphertexts. + +Shares live in memory for the current boot only. + +A keyring failure answers `503` when a retry may help (`store_unavailable`, `mirror_unavailable`, `locked`, `conflict: …`, `dek_unavailable: …`) and `500` when an operator must act (`inconsistent: …`, `escrow_pinned`, `escrow_holders_changed`, `escrow: …`). The reason never contains key material or backend detail; the server log has the rest. + +**Recovery is per node.** Each boot has its own recovery key, so a share wrapped for one node is rejected by another. With several nodes on one database: +- recover one node, by its own address (not through a load balancer); +- then restart the others; a locked node doesn't pick up another node's recovery until it restarts; +- every node on one database must use the same root. A node under another root boots locked, and recovering it re-seals the rows under its root, which locks the others out at their next restart. + +Production runs one node per database. Multi-node recovery is a known limitation. + +### Key continuity + +- **Database restored to an earlier point.** At boot the node takes back a key generation or chain from the block-store copy only once it authenticates: the newest under this node's root, each older one by its chain. During recovery, the recovered key authenticates them. Anything else in the copy (forged, junk, a gap, an oversized object) is skipped with a warning and never stored, so objects planted in the bucket can neither lock the node nor stop it booting while the database is healthy. A block store that refuses the node's own writes is different: with `encrypt` or `strict` it stops the boot (see above). + - Retirement markers (`_tc/at-rest/retired-.json`, written when a holder fix replaces the root key) follow the same rule. A forged or junk marker is skipped. One that can't be read is skipped too, unless it names a key the node has stored or mirrored and the mode is `encrypt` or `strict`: skipping it then could bring a retired key back, so the boot stops until the object reads. With `mode = "off"` a marker never stops the boot. +- **Data keys after a restore.** A space's data key missing from the database is restored from its copy only if its wrap authenticates for that space. A copy that exists but doesn't authenticate is never ignored and never overwritten: it may be the genuine key under a generation the node hasn't loaded (for example after a failed read at boot). That space answers `503 dek_unavailable` until the generation loads or an operator removes the object; no new key is made over it. + - **Runbook, `dek_unavailable` that doesn't clear:** first restart once the block store reads cleanly; a genuine key under an unloaded generation then authenticates and comes back. If the error stays, the server log names the space's data-key object as not authenticating under any loaded generation. Check no newer key generation exists anywhere (database backups, other nodes); only then delete that object under `_tc/at-rest/` and the space gets a new key on its next write. Deleting a genuine key makes everything sealed under it unreadable. +- **No key is ever made over existing material.** The node creates a new root key only when the database has no key, escrow or data-key rows and the block store has nothing under `_tc/at-rest/`. A store or mirror error stops the boot instead, and so does an empty database whose block-store copy holds nothing usable. +- **Escrow and ceremony record after a restore.** A bundle or record the database lost is restored from the block store at boot, so a restore doesn't force a new ceremony: the pinned bundle (the pin authenticates it), or, unpinned, a bundle whose mirrored ceremony record verifies under the key. A record is restored only if its MAC verifies under the key. + - **Runbook, `escrow_holders_changed` you didn't expect:** the server log names the evidence. If it is a block-store object (`_tc/at-rest/escrow-…`) and the configured recipients are right, check it isn't a genuine escrow: compare it with the bundle the holders hold (`tinycloud-at-rest inspect`) and the pinned `efp`. Only then remove that object and restart. A planted object can only stop a boot whose database lost its escrow rows; it never changes who can rebuild the key. +- **Generations.** Each older generation is sealed under the next one's key (its chain), so escrowing the newest generation recovers all of them. At boot under the same root, a missing or broken chain is rebuilt and logged. During recovery under a new root it can't be rebuilt, and the shares are refused with the reason. The newest generation never keeps a chain: a stale one is cleared at boot. +- The key tables' migration `down` refuses while they hold rows. diff --git a/test/m1-realdata-e2e/Cargo.lock b/test/m1-realdata-e2e/Cargo.lock index 4eb7af63..e5d830d2 100644 --- a/test/m1-realdata-e2e/Cargo.lock +++ b/test/m1-realdata-e2e/Cargo.lock @@ -8463,11 +8463,13 @@ dependencies = [ "if-addrs", "lazy_static", "libc", + "lru", "multer", "native-tls", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", + "parking_lot", "percent-encoding", "pin-project", "prometheus", @@ -8493,6 +8495,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tinycloud-core", "tokio", diff --git a/test/w5-policy-runtime-node-e2e/Cargo.lock b/test/w5-policy-runtime-node-e2e/Cargo.lock index 1b1c7b77..f1528b57 100644 --- a/test/w5-policy-runtime-node-e2e/Cargo.lock +++ b/test/w5-policy-runtime-node-e2e/Cargo.lock @@ -8848,11 +8848,13 @@ dependencies = [ "if-addrs", "lazy_static", "libc", + "lru", "multer", "native-tls", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", + "parking_lot", "percent-encoding", "pin-project", "prometheus", @@ -8878,6 +8880,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tinycloud-core", "tokio", diff --git a/tinycloud-node-server/Cargo.toml b/tinycloud-node-server/Cargo.toml index 9a1706ad..1f47ea62 100644 --- a/tinycloud-node-server/Cargo.toml +++ b/tinycloud-node-server/Cargo.toml @@ -55,6 +55,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_with = { version = "3.0", features = ["base64", "hex"] } sha2 = "0.10" +parking_lot = "0.12" +lru = "0.12" serde_ipld_dagcbor.workspace = true toml = "0.8" thiserror = "2.0" @@ -103,6 +105,10 @@ features = ["sqlite", "postgres", "mysql", "tokio"] [dependencies.tinycloud-auth] path = "../tinycloud-auth/" +# TC-887: escrow bundle and recovery wire types for the at-rest routes. +[dependencies.tinycloud-at-rest-escrow] +path = "../tinycloud-at-rest-escrow/" + # TC-409: only for test builds, enable tinycloud-core's opt-in # verification-count test hook so node route tests can assert exactly-once # signature verification. Cargo only activates dev-dependency features when @@ -114,6 +120,8 @@ default-features = false features = ["sqlite", "postgres", "mysql", "tokio", "verification-count-test-hook"] [dev-dependencies] +# TC-887: escrow holders decrypting their shares in tests/at_rest_ceremony.rs. +age = { version = "=0.12.1", default-features = false, features = ["armor"] } # Parse production Rust items in the DID resolver source guard (tests only). syn = { version = "2", features = ["full", "visit"] } quote = "1" diff --git a/tinycloud-node-server/src/at_rest.rs b/tinycloud-node-server/src/at_rest.rs new file mode 100644 index 00000000..238a9fca --- /dev/null +++ b/tinycloud-node-server/src/at_rest.rs @@ -0,0 +1,624 @@ +//! Node wiring for encryption at rest (TC-887 S1): config → keyring, the +//! boot-time escrow gate, and the runtime state the routes use. +//! +//! Two phases, because block storage opens before migrations and the +//! keyring needs its tables: [`AtRestRuntime::new`] before storage, then +//! [`setup`] after migrations installs the keyring. +//! +//! In this build no storage seals anything yet: nothing calls +//! [`AtRestRuntime::declare_seal_path`], so `setup` refuses every mode +//! but `off` and `encrypting()` is always false. +//! +//! With the default config (`mode = "off"`, no recipients, no +//! `escrow_none`) nothing is set up: no key reads, no rows, no mirror. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; + +use anyhow::{bail, ensure, Context, Result}; +use sha2::Digest as _; +use tinycloud_at_rest_escrow::compose::{AppCompose, RECIPIENTS_ENV, VERIFIED_FP_ENV}; +use tinycloud_at_rest_escrow::{Efp, HolderRecipient, RecipientPolicy}; +use tinycloud_core::{ + at_rest::{ + keyring::{GatePass, GateRefusal}, + AtRestKeyring, AtRestStore, DataKeyProvider, KeyMirror, KeyringConfig, KeyringState, + }, + storage::either::Either, + RecoveringConnection, +}; + +use crate::config::{AtRestConfig, AtRestEscrowConfig, AtRestMode, Storage}; +use crate::storage::at_rest_mirror::{FsKeyMirror, S3KeyMirror}; +use crate::tee::TeeContext; + +/// At-rest state shared by the routes and `/info` (and, once it seals +/// writes, block storage). Rocket manages it as `Arc`. +/// +/// The contract for the storage seal path (TC-887 S3), not yet wired: mode +/// `off` behaves exactly as without at-rest. Otherwise writes are sealed +/// only while [`encrypting`](Self::encrypting) (else 503, never plaintext); +/// a sealed read whose key is `Locked` is a 503; legacy plaintext reads are +/// allowed unless the mode is `strict`. +pub struct AtRestRuntime { + mode: AtRestMode, + escrow_none: bool, + keyring: OnceLock>, + /// The gate passed for a non-`off` mode and the keys are unlocked: new + /// writes may be encrypted. Set at boot, or after a recovery unlocks a + /// node whose gate then passes. + encrypting: AtomicBool, + /// Set by block storage when it seals writes (TC-887 S3). Until then a + /// mode other than `off` is refused at boot and `encrypting` stays false. + seal_path: AtomicBool, + share_failures: ShareFailureLimiter, +} + +/// Failed share submissions per client: a share that doesn't decrypt to +/// this boot's recovery key or match a commitment. A client that keeps +/// failing is refused for a while. Failures never spend the keyring's +/// per-boot share budget, which only authenticated shares count against. +/// +/// Bounded: an LRU of [`Self::MAX_CLIENTS`] entries, O(1) per call. When +/// full, the least recently seen client is forgotten (so its budget +/// resets), which only weakens a best-effort limit (TC-1357), never the +/// per-boot budget. +pub struct ShareFailureLimiter { + clients: parking_lot::Mutex>, +} + +impl ShareFailureLimiter { + /// Failures a client may make at once, then one more per minute. + pub const BURST: f64 = 5.0; + const REFILL_PER_SEC: f64 = 1.0 / 60.0; + pub const MAX_CLIENTS: usize = 4096; + + fn new() -> Self { + Self { + clients: parking_lot::Mutex::new(lru::LruCache::new( + std::num::NonZeroUsize::new(Self::MAX_CLIENTS).expect("non-zero"), + )), + } + } + + fn refilled(entry: &mut (f64, std::time::Instant)) { + let now = std::time::Instant::now(); + let elapsed = now.duration_since(entry.1).as_secs_f64(); + entry.0 = (entry.0 + elapsed * Self::REFILL_PER_SEC).min(Self::BURST); + entry.1 = now; + } + + /// Whether `ip` may submit now. + pub fn allows(&self, ip: std::net::IpAddr) -> bool { + let mut clients = self.clients.lock(); + match clients.get_mut(&ip) { + None => true, + Some(entry) => { + Self::refilled(entry); + entry.0 >= 1.0 + } + } + } + + pub fn record_failure(&self, ip: std::net::IpAddr) { + let mut clients = self.clients.lock(); + let entry = clients.get_or_insert_mut(ip, || (Self::BURST, std::time::Instant::now())); + Self::refilled(entry); + entry.0 = (entry.0 - 1.0).max(0.0); + } +} + +/// The node's [`AtRestRuntime`], if it manages one. Unlike a `State` guard it +/// never fails, so a node without it (a test fixture) still launches. +pub struct AtRestState<'r>(pub Option<&'r AtRestRuntime>); + +#[rocket::async_trait] +impl<'r> rocket::request::FromRequest<'r> for AtRestState<'r> { + type Error = std::convert::Infallible; + + async fn from_request( + request: &'r rocket::Request<'_>, + ) -> rocket::request::Outcome { + rocket::request::Outcome::Success(AtRestState( + request + .rocket() + .state::>() + .map(|r| r.as_ref()), + )) + } +} + +impl AtRestRuntime { + /// Phase one, before block storage opens. No keys yet: `provider()` is + /// `None` and `encrypting()` is false until [`setup`] runs. + pub fn new(config: &AtRestConfig) -> Arc { + Arc::new(Self { + mode: config.mode, + escrow_none: config.escrow_none, + keyring: OnceLock::new(), + encrypting: AtomicBool::new(false), + seal_path: AtomicBool::new(false), + share_failures: ShareFailureLimiter::new(), + }) + } + + pub fn mode(&self) -> AtRestMode { + self.mode + } + + /// Whether new writes may be sealed now: storage has a seal path, the + /// gate passed (non-`off` mode) and the keys are unlocked. + pub fn encrypting(&self) -> bool { + self.encrypting.load(Ordering::SeqCst) + } + + /// Storage calls this when it opens with a seal path for every write + /// (the capability S3 turns on). Without it `setup` refuses any mode + /// other than `off`, so the node never claims to encrypt while storage + /// writes plaintext. + pub fn declare_seal_path(&self) { + self.seal_path.store(true, Ordering::SeqCst); + } + + pub fn has_seal_path(&self) -> bool { + self.seal_path.load(Ordering::SeqCst) + } + + pub fn share_failures(&self) -> &ShareFailureLimiter { + &self.share_failures + } + + /// The data-key provider, once [`setup`] installed a keyring. `None` + /// when at-rest isn't configured. A locked keyring is still returned; + /// its calls fail with `DataKeyError::Locked`. + pub fn provider(&self) -> Option> { + self.keyring + .get() + .map(|k| k.clone() as Arc) + } + + pub fn keyring(&self) -> Option<&Arc> { + self.keyring.get() + } + + /// Re-run the gate, e.g. after a recovery unlocked the keys. Only a + /// non-`off` mode can turn encryption on. + pub async fn reevaluate(&self) -> Result { + let Some(keyring) = self.keyring.get() else { + return Err(GateRefusal::NotProvisioned); + }; + let result = keyring.gate(self.escrow_none).await; + let on = self.mode != AtRestMode::Off && self.has_seal_path() && result.is_ok(); + self.encrypting.store(on, Ordering::SeqCst); + result + } + + /// While a mode other than `off` is configured and encryption is off on + /// an unlocked node (for example the gate hit a transient store or + /// mirror error after a recovery), re-run the gate every `every`. + pub fn spawn_gate_retry(self: &Arc, every: std::time::Duration) { + if self.mode == AtRestMode::Off { + return; + } + let runtime = Arc::downgrade(self); + tokio::spawn(async move { + loop { + tokio::time::sleep(every).await; + let Some(runtime) = runtime.upgrade() else { + return; + }; + let unlocked = runtime + .keyring() + .is_some_and(|k| k.state() == KeyringState::Unlocked); + if unlocked && !runtime.encrypting() { + match runtime.reevaluate().await { + Ok(_) if runtime.encrypting() => { + ::tracing::info!( + "at-rest escrow gate passed on retry; encryption enabled" + ) + } + Ok(_) => {} + Err(reason) => { + ::tracing::warn!(%reason, "at-rest escrow gate still closed") + } + } + } + } + }); + } + + pub fn mode_name(&self) -> &'static str { + match self.mode { + AtRestMode::Off => "off", + AtRestMode::Encrypt => "encrypt", + AtRestMode::Strict => "strict", + } + } + + pub fn state_name(&self) -> &'static str { + match self.keyring.get().map(|k| k.state()) { + None => "disabled", + Some(KeyringState::NotProvisioned) => "not_provisioned", + Some(KeyringState::Locked) => "locked", + Some(KeyringState::Unlocked) => "unlocked", + } + } +} + +/// Phase two, after migrations ran: open the keyring and run the gate. +/// +/// Refuses (an error, so the node does not start): +/// - `mode` other than `off` while storage has no seal path +/// ([`AtRestRuntime::declare_seal_path`]); +/// - in a TEE, escrow recipients or a pin that are not literal values in +/// the measured compose ([`check_measured_escrow`]); +/// - `encrypt` or `strict` when the keys are unlocked but the escrow gate +/// fails. +/// +/// A node that boots **locked** starts anyway, with encryption +/// unavailable, so two holders can recover it. +pub async fn setup( + runtime: &AtRestRuntime, + config: &AtRestConfig, + storage: &Storage, + root: &[u8], + db: RecoveringConnection, + sqlite_writer_lock: Option>>, + tee: Option<&TeeContext>, +) -> Result<()> { + let escrow = &config.escrow; + if !enabled(config) { + return Ok(()); + } + if config.mode != AtRestMode::Off && !runtime.has_seal_path() { + bail!( + "at_rest.mode = \"{}\" is refused: this build's storage has no at-rest seal path, so nothing would be encrypted; use mode = \"off\"", + runtime.mode_name() + ); + } + if config.escrow_none && !escrow.recipients.is_empty() { + bail!("set at_rest.escrow.recipients or at_rest.escrow_none, not both"); + } + if tee.is_some() && config.escrow_none { + bail!("at_rest.escrow_none is only for self-hosted Static-root nodes; a TEE node must escrow its key"); + } + if tee.is_some() && escrow.allow_software { + bail!("at_rest.escrow.allow_software is refused inside a TEE: escrow holders must use hardware keys"); + } + // In a TEE every at-rest configuration is checked against the measured + // compose; see `check_tee_escrow`. + if let Some(tee) = tee { + check_tee_escrow(crate::dstack::get_app_compose().await, tee, escrow)?; + } + let pinned_efp = escrow.verified_fp.as_deref().map(parse_efp).transpose()?; + let mirror: Arc = match &storage.blocks { + Either::A(s3) => Arc::new(S3KeyMirror::new(s3).await), + Either::B(_) => Arc::new(FsKeyMirror::new(storage.datadir.join("at-rest"))), + }; + let keyring = AtRestKeyring::open( + AtRestStore::new(db).with_sqlite_writer_lock(sqlite_writer_lock), + mirror, + root, + KeyringConfig { + provision: true, + holders: escrow.recipients.clone(), + recipient_policy: if escrow.allow_software { + RecipientPolicy::AllowSoftware + } else { + RecipientPolicy::HardwareOnly + }, + pinned_efp, + app_id: tee.map(|t| t.app_id.clone()), + compose_hash: tee.map(|t| t.compose_hash.clone()), + // Once keys may seal data, the mirror must hold a copy of every + // key; with mode off, mirror contents never stop the node. + require_mirror: config.mode != AtRestMode::Off, + allow_resplit_before_verify: escrow.allow_resplit_before_verify, + }, + ) + .await + .context("at-rest keys")?; + if runtime.keyring.set(Arc::new(keyring)).is_err() { + bail!("at-rest keys were already set up"); + } + match runtime.keyring().map(|k| k.state()) { + Some(KeyringState::Unlocked) => { + let gate = runtime.reevaluate().await; + match (config.mode, gate) { + (AtRestMode::Off, Ok(GatePass::Verified { .. })) => { + ::tracing::info!("at-rest keys unlocked; escrow verified; mode off") + } + (AtRestMode::Off, Ok(GatePass::Unescrowed)) => ::tracing::info!( + "at-rest keys unlocked; no escrow (escrow_none acknowledged); mode off" + ), + (AtRestMode::Off, Err(refusal)) => ::tracing::warn!( + reason = %refusal, + "at-rest keys unlocked; the escrow gate is not satisfied yet" + ), + (mode, Ok(_)) => ::tracing::info!(?mode, "at-rest encryption enabled"), + (mode, Err(refusal)) => { + bail!("at_rest.mode = {mode:?} is refused: {refusal}") + } + } + } + Some(KeyringState::Locked) => ::tracing::error!( + mode = runtime.mode_name(), + "at-rest keys are LOCKED: encrypted data is unavailable until two escrow holders recover the key (POST /at-rest/recovery/shares)" + ), + _ => bail!("at-rest keys were not provisioned"), + } + Ok(()) +} + +/// Whether any at-rest setting is configured (otherwise nothing is set up). +pub fn enabled(config: &AtRestConfig) -> bool { + config.mode != AtRestMode::Off || !config.escrow.recipients.is_empty() || config.escrow_none +} + +/// The dstack socket exists but its Info couldn't be read, so the node +/// can't tell whether it runs in a TEE. Booting as non-TEE would skip the +/// TEE-only checks (measured compose, no `escrow_none`, no software +/// holders), so with any at-rest setting the node refuses to start. +pub fn refuse_unknown_tee(config: &AtRestConfig, info_error: &anyhow::Error) -> Result<()> { + if enabled(config) { + bail!( + "refusing to start: the dstack socket is present but its Info could not be read ({info_error:#}), so the at-rest TEE checks can't run" + ); + } + Ok(()) +} + +/// The TEE boot check: `app_compose` is what the guest agent returned for +/// `tcb_info.app_compose`. If it couldn't be read, or doesn't parse, or the +/// escrow config doesn't match its literals, the node refuses to start: it +/// never runs at-rest in a TEE without the check. (The guest agent's Info +/// shape is to be confirmed on a live CVM before the escrow ceremony.) +pub fn check_tee_escrow( + app_compose: Result, + tee: &TeeContext, + escrow: &AtRestEscrowConfig, +) -> Result<()> { + let app_compose = app_compose.context( + "refusing to start: at-rest needs the measured app-compose (dstack Info tcb_info.app_compose) to check the escrow config, and it could not be read", + )?; + check_measured_escrow(&app_compose, &tee.compose_hash, escrow).context( + "refusing to start: the at-rest escrow config does not match the measured app-compose", + ) +} + +fn parse_efp(s: &str) -> Result { + s.trim() + .to_ascii_lowercase() + .parse::() + .map_err(|_| anyhow::anyhow!("at_rest.escrow.verified_fp must be 64 hex characters")) +} + +/// In a TEE, the escrow recipients and the pinned fingerprint decide who can +/// rebuild the root key, so they must come from the attested compose: each +/// must equal a literal value in the measured `app-compose.json` (covered by +/// `compose_hash`), never the encrypted environment, a file or any other +/// runtime source. Either one configured without a matching literal, or a +/// literal the effective config doesn't match, is refused. +pub fn check_measured_escrow( + app_compose: &str, + compose_hash: &str, + escrow: &AtRestEscrowConfig, +) -> Result<()> { + let digest = hex::encode(sha2::Sha256::digest(app_compose.as_bytes())); + ensure!( + digest.eq_ignore_ascii_case(compose_hash.trim()), + "the app-compose the guest agent reports hashes to {digest}, not the measured compose_hash {compose_hash}" + ); + let compose = + AppCompose::parse(app_compose.as_bytes()).context("reading the measured app-compose")?; + let literal_recipients = compose.literal_env(RECIPIENTS_ENV)?; + match literal_recipients { + None => ensure!( + escrow.recipients.is_empty(), + "at_rest.escrow.recipients must be set as a literal {RECIPIENTS_ENV} in the measured compose, not from the environment or a file" + ), + Some(literal) => { + let value: rocket::figment::value::Value = + literal.parse().unwrap_or_else(|never| match never {}); + let measured: Vec = value + .deserialize() + .with_context(|| format!("the measured {RECIPIENTS_ENV} does not parse"))?; + ensure!( + measured == escrow.recipients, + "at_rest.escrow.recipients differ from the literal {RECIPIENTS_ENV} in the measured compose" + ); + } + } + let literal_fp = compose.literal_env(VERIFIED_FP_ENV)?; + let configured = escrow.verified_fp.as_deref().map(parse_efp).transpose()?; + match (literal_fp, configured) { + (None, None) => {} + (None, Some(_)) => bail!( + "at_rest.escrow.verified_fp must be set as a literal {VERIFIED_FP_ENV} in the measured compose, not from the environment or a file" + ), + (Some(_), None) => bail!( + "the measured compose pins {VERIFIED_FP_ENV}, but the effective config has no at_rest.escrow.verified_fp" + ), + (Some(literal), Some(configured)) => ensure!( + parse_efp(&literal)? == configured, + "at_rest.escrow.verified_fp differs from the literal {VERIFIED_FP_ENV} in the measured compose" + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const R: &str = "[{holder=\"sam\",recipient=\"age1yubikey1qg3fps8csqk30gjdt552hf7jyjnmt753z9yhk4p8f0x5vvr5m53mju3vr5u\"},{holder=\"hunter\",recipient=\"age1yubikey1qg3fps8csqk30gjdt552hf7jyjnmt753z9yhk4p8f0x5vvr5m53mju3vr5u\"},{holder=\"safe\",recipient=\"age1yubikey1qg3fps8csqk30gjdt552hf7jyjnmt753z9yhk4p8f0x5vvr5m53mju3vr5u\"}]"; + + fn app_compose(env: &str) -> String { + let digest = "a".repeat(64); + serde_json::to_string(&serde_json::json!({ + "manifest_version": 2, + "runner": "docker-compose", + "docker_compose_file": format!( + "services:\n tinycloud:\n image: n@sha256:{digest}\n environment:\n{env}" + ), + })) + .unwrap() + } + + fn hash(s: &str) -> String { + hex::encode(sha2::Sha256::digest(s.as_bytes())) + } + + fn escrow(recipients: &str, fp: Option<&str>) -> AtRestEscrowConfig { + let value: rocket::figment::value::Value = recipients.parse().unwrap(); + AtRestEscrowConfig { + recipients: value.deserialize().unwrap(), + verified_fp: fp.map(str::to_owned), + allow_software: false, + allow_resplit_before_verify: false, + } + } + + #[test] + fn literal_compose_values_pass() { + let fp = "b".repeat(64); + let doc = app_compose(&format!( + " {RECIPIENTS_ENV}: '{R}'\n {VERIFIED_FP_ENV}: {fp}\n" + )); + check_measured_escrow(&doc, &hash(&doc), &escrow(R, Some(&fp.to_uppercase()))).unwrap(); + // No pin yet (before the ceremony): only the recipients. + let doc = app_compose(&format!(" - {RECIPIENTS_ENV}={R}\n")); + check_measured_escrow(&doc, &hash(&doc), &escrow(R, None)).unwrap(); + } + + #[test] + fn escrow_values_from_anywhere_but_the_measured_compose_are_refused() { + let fp = "b".repeat(64); + let cases = [ + // Recipients from the encrypted env (interpolated or passed through). + ( + format!(" {RECIPIENTS_ENV}: ${{RECIPIENTS}}\n"), + escrow(R, None), + "literal", + ), + ( + format!(" - {RECIPIENTS_ENV}\n"), + escrow(R, None), + "literal", + ), + // Configured, but not in the compose at all (env file, config file). + (" A: '1'\n".to_string(), escrow(R, None), "literal"), + // The compose names other holders than the node uses. + ( + format!(" {RECIPIENTS_ENV}: '{}'\n", R.replace("sam", "eve")), + escrow(R, None), + "differ", + ), + // A pin from the environment, or a different one. + ( + format!(" {RECIPIENTS_ENV}: '{R}'\n"), + escrow(R, Some(&fp)), + "verified_fp must be set", + ), + ( + format!( + " {RECIPIENTS_ENV}: '{R}'\n {VERIFIED_FP_ENV}: {}\n", + "c".repeat(64) + ), + escrow(R, Some(&fp)), + "differs", + ), + ( + format!(" {RECIPIENTS_ENV}: '{R}'\n {VERIFIED_FP_ENV}: {fp}\n"), + escrow(R, None), + "no at_rest.escrow.verified_fp", + ), + ]; + for (env, config, needle) in cases { + let doc = app_compose(&env); + let err = format!( + "{:#}", + check_measured_escrow(&doc, &hash(&doc), &config).unwrap_err() + ); + assert!(err.contains(needle), "{env:?}: {err}"); + } + // An app-compose that isn't the measured one. + let doc = app_compose(&format!(" {RECIPIENTS_ENV}: '{R}'\n")); + let err = check_measured_escrow(&doc, &"0".repeat(64), &escrow(R, None)).unwrap_err(); + assert!(err.to_string().contains("measured compose_hash"), "{err}"); + } + + #[test] + fn a_tee_node_refuses_to_start_without_a_readable_measured_compose() { + let doc = app_compose(&format!(" {RECIPIENTS_ENV}: '{R}'\n")); + let tee = TeeContext { + app_id: "a".into(), + compose_hash: hash(&doc), + instance_id: "i".into(), + enforcer_did: "did:key:z".into(), + }; + // The guest agent's Info had no tcb_info.app_compose, or it isn't + // an app-compose at all: never start without the check. + for app_compose in [ + Err(anyhow::anyhow!("dstack Info has no tcb_info")), + Ok("not json".to_string()), + ] { + let err = format!( + "{:#}", + check_tee_escrow(app_compose, &tee, &escrow(R, None)).unwrap_err() + ); + assert!(err.starts_with("refusing to start"), "{err}"); + } + // Readable and matching: the boot may continue. + check_tee_escrow(Ok(doc), &tee, &escrow(R, None)).unwrap(); + } + + #[test] + fn an_unreadable_dstack_info_refuses_at_rest_instead_of_booting_as_non_tee() { + let info_error = anyhow::anyhow!("dstack Info request: connection reset"); + let configured = [ + AtRestConfig { + mode: AtRestMode::Encrypt, + ..Default::default() + }, + AtRestConfig { + escrow: escrow(R, None), + ..Default::default() + }, + AtRestConfig { + escrow_none: true, + ..Default::default() + }, + ]; + for config in configured { + let err = refuse_unknown_tee(&config, &info_error) + .unwrap_err() + .to_string(); + assert!(err.starts_with("refusing to start"), "{err}"); + } + // Nothing at rest configured: the node keeps its old behaviour. + refuse_unknown_tee(&AtRestConfig::default(), &info_error).unwrap(); + } + + #[test] + fn the_share_failure_limiter_stays_bounded_and_fast() { + let limiter = ShareFailureLimiter::new(); + let one: std::net::IpAddr = "203.0.113.7".parse().unwrap(); + for _ in 0..ShareFailureLimiter::BURST as usize { + assert!(limiter.allows(one)); + limiter.record_failure(one); + } + assert!(!limiter.allows(one)); + // 100k distinct clients: O(1) each, and the map never grows past + // its cap (the review measured 10 s per 10k with the old map). + let started = std::time::Instant::now(); + for i in 0..100_000u32 { + limiter.record_failure(std::net::IpAddr::from(i.to_be_bytes())); + } + assert!(started.elapsed() < std::time::Duration::from_secs(2)); + assert_eq!( + limiter.clients.lock().len(), + ShareFailureLimiter::MAX_CLIENTS + ); + } +} diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 9c2746c9..e327d327 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -50,6 +50,54 @@ pub struct Config { pub share_email: ShareEmailConfig, #[serde(default)] pub retention: RetentionConfig, + #[serde(default)] + pub at_rest: AtRestConfig, +} + +/// Encryption at rest (TC-887). Off by default; see `docs/at-rest-escrow.md`. +#[derive(Serialize, Deserialize, Debug, Default, Clone, Hash, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AtRestConfig { + /// `off` writes plaintext; `encrypt` and `strict` are refused at boot + /// unless the escrow gate passes. + #[serde(default)] + pub mode: AtRestMode, + #[serde(default)] + pub escrow: AtRestEscrowConfig, + /// Self-hosted Static-root nodes only: acknowledge that no escrow exists, + /// so losing the root loses encrypted data. Refused inside a TEE. + #[serde(default)] + pub escrow_none: bool, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, Copy, Hash, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum AtRestMode { + #[default] + Off, + Encrypt, + Strict, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, Hash, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AtRestEscrowConfig { + /// Exactly three holders. Setting them provisions the at-rest root key + /// and its escrow bundle at boot, even with `mode = "off"`. + #[serde(default)] + pub recipients: Vec, + /// The escrow fingerprint a ceremony verified (`tinycloud-at-rest record-efp`). + #[serde(default)] + pub verified_fp: Option, + /// Accept X25519 software-key holders (tests and self-host only). + #[serde(default)] + pub allow_software: bool, + /// Before the escrow ceremony only: let a corrected holder list re-split + /// the escrow at boot. The node allows it only while no fingerprint is + /// pinned, the escrow was never verified, and no data key was ever made; + /// otherwise a holder change still refuses the boot. + #[serde(default)] + pub allow_resplit_before_verify: bool, } /// Production exact-email composition. The capability remains unavailable diff --git a/tinycloud-node-server/src/dstack.rs b/tinycloud-node-server/src/dstack.rs index 134da8b4..9e36ebd3 100644 --- a/tinycloud-node-server/src/dstack.rs +++ b/tinycloud-node-server/src/dstack.rs @@ -228,6 +228,30 @@ pub async fn get_info() -> Result { .context("parsing Info response") } +/// The `app-compose.json` this CVM was launched with: `tcb_info.app_compose` +/// from the guest agent's `/Info` (`tcb_info` is a JSON string or object). +/// The caller checks it against `compose_hash`. +pub async fn get_app_compose() -> Result { + let info = get_json(&socket_path(), "/Info") + .await + .context("dstack Info request")?; + app_compose_from_info(&info) +} + +fn app_compose_from_info(info: &serde_json::Value) -> Result { + let tcb = match info.get("tcb_info") { + Some(serde_json::Value::String(s)) => { + serde_json::from_str(s).context("dstack Info tcb_info is not JSON")? + } + Some(other) => other.clone(), + None => return Err(anyhow!("dstack Info has no tcb_info")), + }; + tcb.get("app_compose") + .and_then(|v| v.as_str()) + .map(str::to_owned) + .ok_or_else(|| anyhow!("dstack Info tcb_info has no app_compose")) +} + /// Check whether the dstack socket is reachable. pub fn is_available() -> bool { std::path::Path::new(&socket_path()).exists() @@ -327,6 +351,22 @@ pub(crate) mod stub { mod tests { use super::*; + #[test] + fn app_compose_comes_from_tcb_info_as_a_string_or_an_object() { + let tcb = + serde_json::json!({"mrtd": "00", "app_compose": "{\"runner\":\"docker-compose\"}"}); + let as_string = serde_json::json!({"app_id": "a", "tcb_info": tcb.to_string()}); + let as_object = serde_json::json!({"app_id": "a", "tcb_info": tcb}); + for info in [as_string, as_object] { + assert_eq!( + app_compose_from_info(&info).unwrap(), + "{\"runner\":\"docker-compose\"}" + ); + } + assert!(app_compose_from_info(&serde_json::json!({"app_id": "a"})).is_err()); + assert!(app_compose_from_info(&serde_json::json!({"tcb_info": "{}"})).is_err()); + } + #[tokio::test] async fn get_quote_sends_exactly_64_bytes_of_report_data() { let agent = stub::serve_quote(); diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 8a4c40a1..42221362 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -25,6 +25,7 @@ use std::{ }; pub mod allow_list; +pub mod at_rest; pub mod auth_guards; pub mod authorization; pub mod config; @@ -120,6 +121,10 @@ use routes::tc_bench::{ }; use routes::{ admin::{delete_quota, get_quota, get_usage, list_quotas, set_quota}, + at_rest::{ + at_rest_escrow, at_rest_escrow_record, at_rest_recovery, at_rest_recovery_share, + at_rest_status, + }, attestation::attestation, create_signed_kv_url, delegate, delegation_query, delegation_status, encryption::{ @@ -466,6 +471,11 @@ fn production_routes() -> Vec { public_kv_list, public_kv_options, attestation, + at_rest_status, + at_rest_escrow, + at_rest_escrow_record, + at_rest_recovery, + at_rest_recovery_share, set_quota, delete_quota, get_quota, @@ -587,6 +597,9 @@ pub async fn app_with_control( }) } Err(e) => { + // TC-887: at-rest must not boot as non-TEE when it + // can't tell; that would skip its TEE checks. + at_rest::refuse_unknown_tee(&tinycloud_config.at_rest, &e)?; ::tracing::warn!("dstack socket available but get_info failed: {}", e); None } @@ -633,6 +646,10 @@ pub async fn app_with_control( encryption_backend, ); + // TC-887: at-rest state, before block storage opens so storage can hold + // it; the keyring is installed after migrations (`at_rest::setup`). + let at_rest_runtime = at_rest::AtRestRuntime::new(&tinycloud_config.at_rest); + let tinycloud = TinyCloud::new_with_encryption( database_connection, tinycloud_config.storage.blocks.open().await?, @@ -653,6 +670,19 @@ pub async fn app_with_control( // `database_artifact` table now exists (seeding before migrations would // fail boot on a fresh datadir). Runs before Rocket serves any request. sql_sizes.seed_from(&seed_conn).await?; + // TC-887: at-rest keys (after migrations). Off unless configured; an + // `encrypt`/`strict` mode whose escrow gate fails stops the boot here. + at_rest::setup( + &at_rest_runtime, + &tinycloud_config.at_rest, + &tinycloud_config.storage, + key_setup.as_bytes(), + seed_conn.clone(), + tinycloud.sqlite_writer_lock(), + tee_context.as_ref(), + ) + .await?; + at_rest_runtime.spawn_gate_retry(std::time::Duration::from_secs(30)); #[cfg(feature = "tc-bench-v1")] let bench_state = BenchState::new( &tinycloud_config.tc_bench, @@ -803,6 +833,7 @@ pub async fn app_with_control( .manage(egress_policy) .manage(rate_limiter) .manage(tee_context) + .manage(at_rest_runtime) .manage(encryption_service) .manage(policy_v3_runtime) .manage(seed_conn.clone()) diff --git a/tinycloud-node-server/src/routes/at_rest.rs b/tinycloud-node-server/src/routes/at_rest.rs new file mode 100644 index 00000000..88ab4039 --- /dev/null +++ b/tinycloud-node-server/src/routes/at_rest.rs @@ -0,0 +1,305 @@ +//! Escrow ceremony and recovery endpoints (TC-887 S1). All public data; see +//! `docs/at-rest-escrow.md` and `tinycloud-at-rest` for the holder side. +//! +//! - `GET /at-rest/status`: mode, key state, escrow fingerprint, gate. +//! - `GET /at-rest/escrow`: the escrow bundle holders decrypt from. +//! - `GET /at-rest/escrow/record`: the ceremony verification record, only if +//! its MAC verifies under the live key. +//! - `GET /at-rest/recovery?nonce=`: this boot's recovery recipient, bound +//! with the holder's nonce and the EFP into a TDX quote. +//! - `POST /at-rest/recovery/shares`: one share sealed to that recipient. +//! +//! Share bodies are capped at 4 KiB and rate limited per IP. Shares that +//! don't authenticate (don't decrypt, or match no commitment) are limited +//! per client and never spend the node's per-boot share budget, which only +//! genuine shares count against. Neither share bytes nor ciphertexts are +//! logged. + +use rocket::data::{Data, ToByteUnit}; +use rocket::http::Status; +use rocket::serde::json::Json; +use rocket::State; +use serde::Serialize; +use tinycloud_at_rest_escrow::{ + binding_digest, recovery_binding, validate_nonce, EscrowBundle, RecoveryInfo, + VerificationRecord, MAX_WRAPPED_SHARE, RECOVERY_INFO_VERSION, +}; +use tinycloud_core::at_rest::keyring::{ + AtRestError, EscrowView, KeyringState, ShareOutcome, ShareRefusal, +}; + +use crate::at_rest::{AtRestRuntime, AtRestState}; +use crate::routes::public::{ClientIp, RateLimiter}; +use crate::tee::TeeContext; + +type Refusal = (Status, String); + +fn not_enabled() -> Refusal { + ( + Status::NotFound, + "at-rest keys are not enabled on this node".into(), + ) +} + +/// A keyring error as an answer: 503 when a retry may help (store or mirror +/// unreachable, locked, a concurrent change), 500 when an operator must act. +/// The body is a stable, non-secret reason; backend detail goes to the log. +fn internal(e: AtRestError) -> Refusal { + ::tracing::error!(error = %e, "at-rest keyring error"); + let status = if e.is_transient() { + Status::ServiceUnavailable + } else { + Status::InternalServerError + }; + (status, format!("at-rest keyring error: {}", e.reason())) +} + +async fn view(runtime: &AtRestRuntime) -> Result { + let keyring = runtime.keyring().ok_or_else(not_enabled)?; + keyring.escrow_view().await.map_err(internal)?.ok_or(( + Status::NotFound, + "no escrow bundle is available (no recipients configured, or a locked node without a pinned at_rest.escrow.verified_fp)".into(), + )) +} + +#[derive(Serialize)] +pub struct AtRestStatus { + mode: &'static str, + state: &'static str, + encrypting: bool, + #[serde(skip_serializing_if = "Option::is_none")] + efp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + generation: Option, + verified: bool, +} + +#[get("/at-rest/status")] +pub async fn at_rest_status(at_rest: AtRestState<'_>) -> Result, Refusal> { + let runtime = at_rest.0.ok_or_else(not_enabled)?; + let escrow = match runtime.keyring() { + Some(k) => k.escrow_view().await.map_err(internal)?, + None => None, + }; + Ok(Json(AtRestStatus { + mode: runtime.mode_name(), + state: runtime.state_name(), + encrypting: runtime.encrypting(), + efp: escrow.as_ref().map(|v| v.efp.to_string()), + generation: escrow.as_ref().map(|v| v.generation), + verified: escrow.is_some_and(|v| v.verified), + })) +} + +#[get("/at-rest/escrow")] +pub async fn at_rest_escrow(at_rest: AtRestState<'_>) -> Result, Refusal> { + let runtime = at_rest.0.ok_or_else(not_enabled)?; + Ok(Json(view(runtime).await?.bundle)) +} + +#[get("/at-rest/escrow/record")] +pub async fn at_rest_escrow_record( + at_rest: AtRestState<'_>, +) -> Result, Refusal> { + let runtime = at_rest.0.ok_or_else(not_enabled)?; + let keyring = runtime.keyring().ok_or_else(not_enabled)?; + keyring + .verification_record() + .await + .map_err(internal)? + .map(Json) + .ok_or(( + Status::NotFound, + "no verified ceremony record for the active escrow".into(), + )) +} + +#[get("/at-rest/recovery?")] +pub async fn at_rest_recovery( + at_rest: AtRestState<'_>, + tee: &State>, + client_ip: ClientIp, + rate_limiter: &State, + nonce: String, +) -> Result, Refusal> { + rate_limiter + .check(client_ip.0) + .map_err(|s| (s, "Rate limit exceeded".to_string()))?; + validate_nonce(&nonce).map_err(|e| (Status::BadRequest, e.to_string()))?; + let runtime = at_rest.0.ok_or_else(not_enabled)?; + let view = view(runtime).await?; + let binding = recovery_binding(&view.recovery_recipient, &view.efp, &nonce); + let digest = binding_digest(&binding); + let (mode, quote, event_log) = quote_binding(tee.inner().as_ref(), &digest.0).await?; + Ok(Json(RecoveryInfo { + version: RECOVERY_INFO_VERSION, + mode: mode.into(), + state: match view.state { + KeyringState::Locked => "locked", + _ => "unlocked", + } + .into(), + ark_id: view.ark_id, + generation: view.generation, + efp: view.efp, + canary_fp: view.canary_fp, + recovery_recipient: view.recovery_recipient, + binding, + binding_sha256: digest, + quote, + event_log, + shares_held: view.shares_held, + })) +} + +/// A TDX quote over `digest`, or `classic` with no quote outside a TEE (a +/// holder's tool refuses to release a share then). +async fn quote_binding( + tee: Option<&TeeContext>, + digest: &[u8; 32], +) -> Result<(&'static str, Option, Option), Refusal> { + match tee { + #[cfg(feature = "dstack")] + Some(_) => { + let quote = crate::dstack::get_quote(&crate::dstack::socket_path(), digest) + .await + .map_err(|e| { + ::tracing::error!(error_class = %e, "TDX quote unavailable for at-rest recovery"); + ( + Status::ServiceUnavailable, + "quote generation failed".to_string(), + ) + })?; + Ok(("dstack", Some(quote.quote), Some(quote.event_log))) + } + _ => { + let _ = digest; + Ok(("classic", None, None)) + } + } +} + +#[derive(Debug, Serialize)] +pub struct ShareAnswer { + outcome: &'static str, + shares_held: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + efp: Option, + /// After a recovery: whether encryption is enabled again (the gate + /// passed), or why not. + #[serde(skip_serializing_if = "Option::is_none")] + gate: Option, +} + +#[post("/at-rest/recovery/shares", data = "")] +pub async fn at_rest_recovery_share( + at_rest: AtRestState<'_>, + // TC-1357: ClientIp trusts the proxy IP header from any peer; per-client + // limits are best effort until it is trusted only from the ingress. + client_ip: ClientIp, + rate_limiter: &State, + body: Data<'_>, +) -> Result, Refusal> { + rate_limiter + .check(client_ip.0) + .map_err(|s| (s, "Rate limit exceeded".to_string()))?; + let runtime = at_rest.0.ok_or_else(not_enabled)?; + let keyring = runtime.keyring().ok_or_else(not_enabled)?; + let failures = runtime.share_failures(); + if !failures.allows(client_ip.0) { + return Err(( + Status::TooManyRequests, + "too many rejected shares from this client; retry later".into(), + )); + } + let wrapped = body + .open(MAX_WRAPPED_SHARE.bytes()) + .into_bytes() + .await + .map_err(|_| { + failures.record_failure(client_ip.0); + (Status::BadRequest, "could not read the share".to_string()) + })?; + if !wrapped.is_complete() { + failures.record_failure(client_ip.0); + return Err(( + Status::PayloadTooLarge, + format!("a wrapped share is at most {MAX_WRAPPED_SHARE} bytes"), + )); + } + match keyring.submit_share(&wrapped).await { + Ok(ShareOutcome::Held { shares_held }) => Ok(Json(ShareAnswer { + outcome: "held", + shares_held, + efp: None, + gate: None, + })), + Ok(ShareOutcome::Verified { efp }) => { + let gate = runtime.reevaluate().await; + Ok(Json(ShareAnswer { + outcome: "verified", + shares_held: vec![], + efp: Some(efp.to_string()), + gate: Some(match gate { + Ok(_) => "open".into(), + Err(r) => r.to_string(), + }), + })) + } + Ok(ShareOutcome::Unlocked) => { + let gate = match runtime.reevaluate().await { + Ok(_) => "open".to_string(), + Err(r) => { + ::tracing::error!(reason = %r, "at-rest keys recovered, but encryption stays off"); + r.to_string() + } + }; + Ok(Json(ShareAnswer { + outcome: "unlocked", + shares_held: vec![], + efp: None, + gate: Some(gate), + })) + } + Err(ShareRefusal::RateLimited) => Err(( + Status::TooManyRequests, + "too many share submissions this boot; retry later".into(), + )), + Err(ShareRefusal::NoBundle(why)) => Err((Status::Conflict, why.into())), + Err(ShareRefusal::Invalid(e)) => { + failures.record_failure(client_ip.0); + Err((Status::BadRequest, format!("share rejected: {e}"))) + } + Err(ShareRefusal::Keyring(e)) => Err(internal(e)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keyring_errors_answer_with_a_status_and_a_non_secret_reason() { + let (status, body) = internal(AtRestError::Mirror( + "s3://bucket/secret-path: access denied".into(), + )); + assert_eq!(status, Status::ServiceUnavailable); + assert_eq!(body, "at-rest keyring error: mirror_unavailable"); + let (status, body) = internal(AtRestError::Inconsistent( + "an older ARK generation's chain is missing or does not open", + )); + assert_eq!(status, Status::InternalServerError); + assert!( + body.contains("inconsistent: an older ARK generation"), + "{body}" + ); + let (status, body) = internal(AtRestError::EscrowHoldersChanged); + assert_eq!( + (status, body.as_str()), + ( + Status::InternalServerError, + "at-rest keyring error: escrow_holders_changed" + ) + ); + } +} diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index b1fa77c4..a6c8ad83 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -65,6 +65,7 @@ use tinycloud_core::{ }; pub mod admin; +pub mod at_rest; pub mod attestation; pub mod encryption; pub mod hooks; @@ -206,6 +207,18 @@ pub struct NodeInfo { /// when the node's database is not PostgreSQL. #[serde(skip_serializing_if = "Option::is_none")] pub postgres: Option, + /// Encryption at rest (TC-887): omitted unless the node is configured + /// for it. Details at `/at-rest/status`. + #[serde(skip_serializing_if = "Option::is_none")] + pub at_rest: Option, +} + +#[derive(Serialize)] +pub struct AtRestInfo { + pub mode: &'static str, + /// `locked` means encrypted data is unavailable until recovery. + pub state: &'static str, + pub encrypting: bool, } /// Process-wide PostgreSQL connection counters in [`NodeInfo`]. @@ -222,6 +235,7 @@ fn build_info( quota_cache: &State, encryption: &State, database: Option<&tinycloud_core::RecoveringConnection>, + at_rest: Option<&crate::at_rest::AtRestRuntime>, ) -> NodeInfo { #[allow(unused_mut)] let mut features = vec!["kv", "delegation", "sharing", "sql"]; @@ -257,6 +271,13 @@ fn build_info( sockets_configured: sqlx_core::net::tcp_keepalive::sockets_configured(), connections_opened: crate::postgres_connections_opened(), }), + at_rest: at_rest + .filter(|r| r.keyring().is_some()) + .map(|r| AtRestInfo { + mode: r.mode_name(), + state: r.state_name(), + encrypting: r.encrypting(), + }), } } @@ -285,8 +306,9 @@ pub fn info( quota_cache: &State, encryption: &State, pool: NodePool<'_>, + at_rest: crate::at_rest::AtRestState<'_>, ) -> Json { - Json(build_info(tee, quota_cache, encryption, pool.0)) + Json(build_info(tee, quota_cache, encryption, pool.0, at_rest.0)) } #[get("/version")] @@ -295,8 +317,9 @@ pub fn version( quota_cache: &State, encryption: &State, pool: NodePool<'_>, + at_rest: crate::at_rest::AtRestState<'_>, ) -> Json { - Json(build_info(tee, quota_cache, encryption, pool.0)) + Json(build_info(tee, quota_cache, encryption, pool.0, at_rest.0)) } #[allow(clippy::let_unit_value)] diff --git a/tinycloud-node-server/src/storage/at_rest_mirror.rs b/tinycloud-node-server/src/storage/at_rest_mirror.rs new file mode 100644 index 00000000..f97bd34b --- /dev/null +++ b/tinycloud-node-server/src/storage/at_rest_mirror.rs @@ -0,0 +1,360 @@ +//! Where the at-rest keyring mirrors its sealed ARK rows and escrow bundles +//! (TC-887): next to the blocks, so a database restored to before an ARK +//! row was written can still open blocks stored after it. +//! +//! - S3: objects under `_tc/at-rest/` in the block bucket. The boot size scan +//! only counts `{space}/{hash}` keys, so these are never charged to a space. +//! - Filesystem: files under `{datadir}/at-rest/`, outside the block +//! directory (whose size scan treats every subdirectory as a DID). + +use std::path::{Component, Path, PathBuf}; + +use aws_sdk_s3::{ + error::{GetObjectError, GetObjectErrorKind}, + types::{ByteStream, SdkError}, + Client, +}; +use futures::TryStreamExt; +use tinycloud_core::at_rest::{ + store::{MirrorError, MirrorObjectTooLarge, MAX_MIRROR_OBJECT, MIRROR_PREFIX}, + KeyMirror, +}; + +use super::s3::{new_client, S3BlockConfig}; + +/// Control objects in the S3 block bucket. +pub struct S3KeyMirror { + client: Client, + bucket: String, +} + +impl S3KeyMirror { + pub async fn new(config: &S3BlockConfig) -> Self { + Self { + client: new_client(config).await, + bucket: config.bucket.clone(), + } + } +} + +fn check_name(name: &str) -> Result<(), MirrorError> { + let ok = name.starts_with(MIRROR_PREFIX) + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b'/')) + && !name.contains(".."); + if ok { + Ok(()) + } else { + Err(format!("invalid at-rest mirror name {name:?}").into()) + } +} + +#[async_trait::async_trait] +impl KeyMirror for S3KeyMirror { + async fn put(&self, name: &str, bytes: Vec) -> Result<(), MirrorError> { + check_name(name)?; + self.client + .put_object() + .bucket(&self.bucket) + .key(name) + .body(ByteStream::from(bytes)) + .send() + .await + .map_err(|e| aws_sdk_s3::Error::from(e).into()) + .map(|_| ()) + } + + async fn get(&self, name: &str) -> Result>, MirrorError> { + check_name(name)?; + match self + .client + .get_object() + .bucket(&self.bucket) + .key(name) + .send() + .await + { + Ok(o) => { + // A key object is a few hundred bytes; anything far larger + // is junk planted in the bucket, never read into memory. + if o.content_length() > MAX_MIRROR_OBJECT as i64 { + return Err(Box::new(MirrorObjectTooLarge)); + } + let mut body = o.body; + let mut out = Vec::new(); + while let Some(chunk) = body.try_next().await? { + if out.len() + chunk.len() > MAX_MIRROR_OBJECT { + return Err(Box::new(MirrorObjectTooLarge)); + } + out.extend_from_slice(&chunk); + } + Ok(Some(out)) + } + Err(SdkError::ServiceError { + err: + GetObjectError { + kind: GetObjectErrorKind::NoSuchKey(_), + .. + }, + .. + }) => Ok(None), + Err(e) => Err(aws_sdk_s3::Error::from(e).into()), + } + } + + async fn list(&self, prefix: &str) -> Result, MirrorError> { + check_name(prefix)?; + let pages = self + .client + .list_objects_v2() + .bucket(&self.bucket) + .prefix(prefix) + .into_paginator() + .send(); + let names = pages + .map_err(|e| -> MirrorError { aws_sdk_s3::Error::from(e).into() }) + .try_fold(Vec::new(), |mut acc, page| async move { + acc.extend( + page.contents + .into_iter() + .flatten() + .filter_map(|o| o.key().map(str::to_owned)) + // A name `get` would refuse can't be a key object; + // listing it would let a bucket writer fail reads. + .filter(|k| check_name(k).is_ok()), + ); + Ok(acc) + }) + .await?; + Ok(names) + } + + async fn delete(&self, name: &str) -> Result<(), MirrorError> { + check_name(name)?; + // S3 answers success for a key that doesn't exist. + self.client + .delete_object() + .bucket(&self.bucket) + .key(name) + .send() + .await + .map_err(|e| aws_sdk_s3::Error::from(e).into()) + .map(|_| ()) + } +} + +/// Control files under a local directory. +pub struct FsKeyMirror { + root: PathBuf, +} + +impl FsKeyMirror { + pub fn new(root: PathBuf) -> Self { + Self { root } + } + + fn path(&self, name: &str) -> Result { + check_name(name)?; + let rel = Path::new(name); + if rel.components().any(|c| !matches!(c, Component::Normal(_))) { + return Err(format!("invalid at-rest mirror name {name:?}").into()); + } + Ok(self.root.join(rel)) + } +} + +#[async_trait::async_trait] +impl KeyMirror for FsKeyMirror { + async fn put(&self, name: &str, bytes: Vec) -> Result<(), MirrorError> { + let path = self.path(name)?; + let dir = path.parent().ok_or("mirror path has no parent")?; + tokio::fs::create_dir_all(dir).await?; + // Write, fsync, then rename, so a crash never leaves a half-written + // or unsynced key file under the final name. The temporary name is + // unique, so concurrent writers of one object never collide. + let file_name = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or("mirror path has no file name")?; + let tmp = dir.join(format!("{file_name}.{:016x}.tmp", rand::random::())); + let mut file = tokio::fs::File::create(&tmp).await?; + tokio::io::AsyncWriteExt::write_all(&mut file, &bytes).await?; + file.sync_all().await?; + drop(file); + tokio::fs::rename(&tmp, &path).await?; + tokio::fs::File::open(dir).await?.sync_all().await?; + Ok(()) + } + + /// A regular file only: a symlink is never followed (`O_NOFOLLOW`), the + /// open never blocks (`O_NONBLOCK`, so a FIFO can't hang it), and a + /// directory, FIFO or other non-file counts as absent (checked with + /// `fstat` on the open handle). The read is bounded by + /// [`MAX_MIRROR_OBJECT`] whatever the file claims its size is. + async fn get(&self, name: &str) -> Result>, MirrorError> { + use tokio::io::AsyncReadExt as _; + let path = self.path(name)?; + let file = tokio::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(&path) + .await; + let file = match file { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) if e.raw_os_error() == Some(libc::ELOOP) => return Ok(None), + Err(e) => return Err(e.into()), + }; + if !file.metadata().await?.is_file() { + return Ok(None); + } + let mut bytes = Vec::new(); + file.take(MAX_MIRROR_OBJECT as u64 + 1) + .read_to_end(&mut bytes) + .await?; + if bytes.len() > MAX_MIRROR_OBJECT { + return Err(Box::new(MirrorObjectTooLarge)); + } + Ok(Some(bytes)) + } + + async fn list(&self, prefix: &str) -> Result, MirrorError> { + check_name(prefix)?; + // Names are "_tc/at-rest/": one directory level to scan. + let (dir_name, file_prefix) = prefix.rsplit_once('/').ok_or("prefix has no directory")?; + let dir = self.path(&format!("{dir_name}/"))?; + let mut entries = match tokio::fs::read_dir(&dir).await { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e.into()), + }; + let mut names = Vec::new(); + while let Some(entry) = entries.next_entry().await? { + // Regular files only (`file_type` doesn't follow symlinks), and + // only names `get` accepts. + if !entry.file_type().await?.is_file() { + continue; + } + if let Ok(file) = entry.file_name().into_string() { + let name = format!("{dir_name}/{file}"); + if file.starts_with(file_prefix) + && !file.ends_with(".tmp") + && check_name(&name).is_ok() + { + names.push(name); + } + } + } + Ok(names) + } + + async fn delete(&self, name: &str) -> Result<(), MirrorError> { + let path = self.path(name)?; + match tokio::fs::remove_file(&path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn fs_mirror_round_trips_lists_and_refuses_escapes() { + let dir = tempfile::tempdir().unwrap(); + let m = FsKeyMirror::new(dir.path().to_path_buf()); + assert!(m.list("_tc/at-rest/ark-").await.unwrap().is_empty()); + m.put("_tc/at-rest/ark-0000000001-ab.json", b"one".to_vec()) + .await + .unwrap(); + m.put("_tc/at-rest/escrow-cd.json", b"two".to_vec()) + .await + .unwrap(); + assert_eq!( + m.get("_tc/at-rest/ark-0000000001-ab.json").await.unwrap(), + Some(b"one".to_vec()) + ); + assert_eq!(m.get("_tc/at-rest/missing.json").await.unwrap(), None); + assert_eq!( + m.list("_tc/at-rest/ark-").await.unwrap(), + vec!["_tc/at-rest/ark-0000000001-ab.json".to_string()] + ); + m.delete("_tc/at-rest/ark-0000000001-ab.json") + .await + .unwrap(); + m.delete("_tc/at-rest/ark-0000000001-ab.json") + .await + .unwrap(); + assert!(m.list("_tc/at-rest/ark-").await.unwrap().is_empty()); + assert!(m.delete("../x").await.is_err()); + for bad in [ + "../x", + "_tc/at-rest/../../etc", + "other/x", + "_tc/at-rest/a b", + ] { + assert!(m.put(bad, vec![]).await.is_err(), "{bad}"); + } + } + + #[tokio::test] + async fn fs_mirror_lists_only_readable_regular_files_and_never_follows_links() { + let dir = tempfile::tempdir().unwrap(); + let m = FsKeyMirror::new(dir.path().to_path_buf()); + m.put("_tc/at-rest/ark-0000000001-ab.json", b"one".to_vec()) + .await + .unwrap(); + let at_rest = dir.path().join("_tc/at-rest"); + // Planted by a datadir writer: names `get` refuses, a directory, a + // symlink to a huge stream and one to a secret file. + for name in [ + "ark-0000000002-x y.json", + "ark-..json", + "ark-0000000002-x+y.json", + ] { + std::fs::write(at_rest.join(name), b"junk").unwrap(); + } + std::fs::create_dir(at_rest.join("ark-0000000002-d.json")).unwrap(); + std::os::unix::fs::symlink("/dev/zero", at_rest.join("ark-0000000002-z.json")).unwrap(); + let secret = dir.path().join("secret"); + std::fs::write(&secret, b"not a mirror object").unwrap(); + std::os::unix::fs::symlink(&secret, at_rest.join("ark-0000000002-s.json")).unwrap(); + // A FIFO under a name the node reads directly (escrow and record + // names are public): opening it used to block forever. + let fifo = + std::ffi::CString::new(at_rest.join("escrow-ff.json").to_str().unwrap().as_bytes()) + .unwrap(); + assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o600) }, 0); + + assert_eq!( + m.list("_tc/at-rest/ark-").await.unwrap(), + vec!["_tc/at-rest/ark-0000000001-ab.json".to_string()] + ); + for planted in [ + "_tc/at-rest/ark-0000000002-d.json", + "_tc/at-rest/ark-0000000002-z.json", + "_tc/at-rest/ark-0000000002-s.json", + "_tc/at-rest/escrow-ff.json", + ] { + let got = tokio::time::timeout(std::time::Duration::from_secs(5), m.get(planted)) + .await + .unwrap_or_else(|_| panic!("{planted}: get hung")); + assert_eq!(got.unwrap(), None, "{planted}"); + } + // A regular file over the cap is refused without reading it all. + std::fs::write( + at_rest.join("ark-0000000003-big.json"), + vec![b' '; MAX_MIRROR_OBJECT + 1], + ) + .unwrap(); + let err = m + .get("_tc/at-rest/ark-0000000003-big.json") + .await + .unwrap_err(); + assert!(err.is::(), "{err}"); + } +} diff --git a/tinycloud-node-server/src/storage/mod.rs b/tinycloud-node-server/src/storage/mod.rs index df299470..80d92074 100644 --- a/tinycloud-node-server/src/storage/mod.rs +++ b/tinycloud-node-server/src/storage/mod.rs @@ -1,3 +1,4 @@ +pub mod at_rest_mirror; pub mod file_system; pub mod s3; pub mod size; diff --git a/tinycloud-node-server/src/storage/s3.rs b/tinycloud-node-server/src/storage/s3.rs index 8f9abe27..932baf84 100644 --- a/tinycloud-node-server/src/storage/s3.rs +++ b/tinycloud-node-server/src/storage/s3.rs @@ -56,7 +56,7 @@ impl StorageSetup for S3BlockStore { } } -async fn new_client(config: &S3BlockConfig) -> Client { +pub(crate) async fn new_client(config: &S3BlockConfig) -> Client { let general_config = aws_config().await; let sdk_config = aws_sdk_s3::config::Builder::from(&general_config); let sdk_config = match &config.endpoint { diff --git a/tinycloud-node-server/tests/at_rest_ceremony.rs b/tinycloud-node-server/tests/at_rest_ceremony.rs new file mode 100644 index 00000000..0e9de195 --- /dev/null +++ b/tinycloud-node-server/tests/at_rest_ceremony.rs @@ -0,0 +1,372 @@ +//! TC-887 S1 through the real app: provisioning, the ceremony test-unlock, +//! the encrypt gate at boot, and recovery of a node booted under a new root. +//! SQLite file and filesystem blocks in a temp dir, so restarts keep state. + +use std::io::Read; +use std::net::SocketAddr; +use std::path::Path; + +use rocket::{ + figment::{providers::Serialized, Figment}, + http::{ContentType, Status}, + local::asynchronous::Client, +}; +use serde_json::{json, Value}; +use tinycloud_at_rest_escrow::{ + binding_digest, recovery_binding, wrap_share, EscrowBundle, RecoveryInfo, +}; + +const NONCE: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + +struct Holders(Vec); + +impl Holders { + fn new() -> Self { + Self((0..3).map(|_| age::x25519::Identity::generate()).collect()) + } + + fn recipients(&self) -> Value { + json!(["sam", "hunter", "safe"] + .iter() + .zip(&self.0) + .map(|(h, id)| json!({"holder": h, "recipient": id.to_public().to_string()})) + .collect::>()) + } + + /// Holder `i`: `age -d` their share, then seal it to the node's recovery key. + fn wrapped(&self, bundle: &EscrowBundle, info: &RecoveryInfo, i: usize) -> String { + let armored = &bundle.shares[i].ciphertext; + let decryptor = + age::Decryptor::new_buffered(age::armor::ArmoredReader::new(armored.as_bytes())) + .unwrap(); + let mut plain = Vec::new(); + decryptor + .decrypt(std::iter::once(&self.0[i] as &dyn age::Identity)) + .unwrap() + .read_to_end(&mut plain) + .unwrap(); + let share = bundle.open_share(&plain).unwrap(); + wrap_share(&info.recovery_recipient, &share).unwrap() + } +} + +async fn app(dir: &Path, root: u8, at_rest: Value) -> anyhow::Result { + let mut config = tinycloud::config::Config::default(); + config.storage.datadir = dir.into(); + config.storage.database = Some(format!("sqlite:{}?mode=rwc", dir.join("caps.db").display())); + config.keys = serde_json::from_value(json!({ + "type": "static", + "secret": base64::encode_config([root; 32], base64::URL_SAFE_NO_PAD), + }))?; + config.at_rest = serde_json::from_value(at_rest)?; + let figment = Figment::from(rocket::Config { + address: "127.0.0.1".parse()?, + log_level: rocket::config::LogLevel::Off, + ..rocket::Config::default() + }) + .merge(Serialized::defaults(&config)); + let rocket = tinycloud::app(&figment).await?; + Ok(Client::untracked(rocket).await?) +} + +/// The boot must fail; returns the full error chain. +async fn refused(dir: &Path, root: u8, at_rest: Value) -> String { + match app(dir, root, at_rest).await { + Ok(_) => panic!("the node started, but this configuration must be refused"), + Err(e) => format!("{e:#}"), + } +} + +fn remote() -> SocketAddr { + "127.0.0.1:40000".parse().unwrap() +} + +async fn get_json(client: &Client, uri: &str) -> (Status, Value) { + let res = client + .get(uri.to_string()) + .remote(remote()) + .dispatch() + .await; + let status = res.status(); + let body = res.into_string().await.unwrap_or_default(); + ( + status, + serde_json::from_str(&body).unwrap_or(Value::String(body)), + ) +} + +async fn post_share(client: &Client, body: impl Into>) -> (Status, Value) { + let res = client + .post("/at-rest/recovery/shares") + .remote(remote()) + .header(ContentType::Plain) + .body(body.into()) + .dispatch() + .await; + let status = res.status(); + let body = res.into_string().await.unwrap_or_default(); + ( + status, + serde_json::from_str(&body).unwrap_or(Value::String(body)), + ) +} + +async fn recovery(client: &Client) -> RecoveryInfo { + let (status, body) = get_json(client, &format!("/at-rest/recovery?nonce={NONCE}")).await; + assert_eq!(status, Status::Ok, "{body}"); + let info: RecoveryInfo = serde_json::from_value(body).unwrap(); + // No TEE here: no quote, and the holder tool would refuse to release. + assert_eq!(info.mode, "classic"); + assert!(info.quote.is_none()); + assert_eq!( + info.binding, + recovery_binding(&info.recovery_recipient, &info.efp, NONCE) + ); + assert_eq!(info.binding_sha256, binding_digest(&info.binding)); + info +} + +#[tokio::test] +async fn ceremony_gate_and_recovery_through_the_app() { + let dir = tempfile::tempdir().unwrap(); + let h = Holders::new(); + let escrow = |fp: Option<&str>| json!({"recipients": h.recipients(), "allow_software": true, "verified_fp": fp}); + + // Default config: nothing at rest, and /info says nothing about it. + let client = app(dir.path(), 1, json!({})).await.unwrap(); + let (_, info) = get_json(&client, "/info").await; + assert!(info.get("at_rest").is_none(), "{info}"); + let (_, status) = get_json(&client, "/at-rest/status").await; + assert_eq!(status["state"], "disabled"); + assert_eq!( + get_json(&client, "/at-rest/escrow").await.0, + Status::NotFound + ); + drop(client); + assert!(!dir.path().join("at-rest").exists()); + + // Release A: recipients pinned, mode off. The ARK and escrow are made. + let client = app(dir.path(), 1, json!({"escrow": escrow(None)})) + .await + .unwrap(); + let (_, info) = get_json(&client, "/info").await; + assert_eq!( + info["at_rest"], + json!({"mode":"off","state":"unlocked","encrypting":false}) + ); + let (_, bundle) = get_json(&client, "/at-rest/escrow").await; + let bundle: EscrowBundle = serde_json::from_value(bundle).unwrap(); + let efp = bundle.fingerprint().to_string(); + assert!(std::fs::read_dir(dir.path().join("at-rest/_tc/at-rest")) + .unwrap() + .any(|e| e.unwrap().file_name().to_string_lossy().starts_with("ark-"))); + + // Ceremony test-unlock: junk and oversize are refused, two shares are + // held, the third verifies. + assert_eq!(post_share(&client, "junk").await.0, Status::BadRequest); + assert_eq!( + post_share(&client, vec![b'a'; 8192]).await.0, + Status::PayloadTooLarge + ); + let info = recovery(&client).await; + assert_eq!(info.state, "unlocked"); + let (s, a) = post_share(&client, h.wrapped(&bundle, &info, 0)).await; + assert_eq!( + (s, &a["outcome"], &a["shares_held"]), + (Status::Ok, &json!("held"), &json!([1])) + ); + let (_, a) = post_share(&client, h.wrapped(&bundle, &info, 2)).await; + assert_eq!(a["shares_held"], json!([1, 3])); + assert_eq!(recovery(&client).await.shares_held, vec![1, 3]); + let (_, a) = post_share(&client, h.wrapped(&bundle, &info, 1)).await; + assert_eq!(a["outcome"], "verified"); + assert_eq!(a["efp"], json!(efp)); + let (status, record) = get_json(&client, "/at-rest/escrow/record").await; + assert_eq!(status, Status::Ok); + assert_eq!(record["efp"], json!(efp)); + let (_, st) = get_json(&client, "/at-rest/status").await; + assert_eq!( + (st["verified"].clone(), st["encrypting"].clone()), + (json!(true), json!(false)) + ); + drop(client); + + // No seal path in this build: encrypt and strict are refused at boot, + // even with the escrow pinned and verified, so nothing ever claims to + // encrypt while storage writes plaintext. + for mode in ["encrypt", "strict"] { + let err = refused( + dir.path(), + 1, + json!({"mode": mode, "escrow": escrow(Some(&efp))}), + ) + .await; + assert!(err.contains("no at-rest seal path"), "{err}"); + } + + // Pinned and verified, mode off: the gate passes, nothing is encrypted. + let client = app(dir.path(), 1, json!({"escrow": escrow(Some(&efp))})) + .await + .unwrap(); + let (_, info) = get_json(&client, "/info").await; + assert_eq!( + info["at_rest"], + json!({"mode":"off","state":"unlocked","encrypting":false}) + ); + drop(client); + + // New root (new app id / KMS): boots LOCKED, not refused. + let client = app(dir.path(), 2, json!({"escrow": escrow(Some(&efp))})) + .await + .unwrap(); + let (_, info) = get_json(&client, "/info").await; + assert_eq!( + info["at_rest"], + json!({"mode":"off","state":"locked","encrypting":false}) + ); + let (status, _) = get_json(&client, "/at-rest/escrow/record").await; + assert_eq!(status, Status::NotFound); + let info = recovery(&client).await; + assert_eq!(info.state, "locked"); + // One share, even twice, does nothing. + let one = h.wrapped(&bundle, &info, 1); + assert_eq!( + post_share(&client, one.clone()).await.1["shares_held"], + json!([2]) + ); + assert_eq!(post_share(&client, one).await.1["shares_held"], json!([2])); + let (_, st) = get_json(&client, "/at-rest/status").await; + assert_eq!(st["state"], "locked"); + // A second holder unlocks it and the escrow gate passes again. + let (_, a) = post_share(&client, h.wrapped(&bundle, &info, 2)).await; + assert_eq!( + (a["outcome"].clone(), a["gate"].clone()), + (json!("unlocked"), json!("open")) + ); + let (_, info) = get_json(&client, "/info").await; + assert_eq!( + info["at_rest"], + json!({"mode":"off","state":"unlocked","encrypting":false}) + ); + drop(client); + + // Resealed under the new root: it now boots unlocked. + let client = app(dir.path(), 2, json!({"escrow": escrow(Some(&efp))})) + .await + .unwrap(); + let (_, info) = get_json(&client, "/info").await; + assert_eq!(info["at_rest"]["state"], json!("unlocked")); +} + +async fn post_share_from(client: &Client, ip: &str, body: impl Into>) -> (Status, Value) { + let res = client + .post("/at-rest/recovery/shares") + .remote(format!("{ip}:4000").parse().unwrap()) + .header(ContentType::Plain) + .body(body.into()) + .dispatch() + .await; + let status = res.status(); + let body = res.into_string().await.unwrap_or_default(); + ( + status, + serde_json::from_str(&body).unwrap_or(Value::String(body)), + ) +} + +/// The review's starvation probe, inverted: junk from one client, at about +/// ten requests a minute, never keeps a genuine holder from recovering a +/// locked node. +#[tokio::test] +async fn junk_from_one_client_does_not_block_a_genuine_share() { + let dir = tempfile::tempdir().unwrap(); + let h = Holders::new(); + let escrow = |fp: Option<&str>| json!({"recipients": h.recipients(), "allow_software": true, "verified_fp": fp}); + let client = app(dir.path(), 1, json!({"escrow": escrow(None)})) + .await + .unwrap(); + let (_, bundle) = get_json(&client, "/at-rest/escrow").await; + let bundle: EscrowBundle = serde_json::from_value(bundle).unwrap(); + let efp = bundle.fingerprint().to_string(); + drop(client); + + let client = app(dir.path(), 2, json!({"escrow": escrow(Some(&efp))})) + .await + .unwrap(); + let info = recovery(&client).await; + assert_eq!(info.state, "locked"); + // Ten junk bodies from one address: rejected, then that client is + // throttled; the node's share budget is untouched. + let mut statuses = Vec::new(); + for _ in 0..10 { + statuses.push(post_share_from(&client, "203.0.113.7", "junk").await.0); + } + assert!(statuses.contains(&Status::BadRequest), "{statuses:?}"); + assert_eq!(statuses.last(), Some(&Status::TooManyRequests)); + // Two genuine holders from other addresses recover the node. + let (s, a) = post_share_from(&client, "198.51.100.1", h.wrapped(&bundle, &info, 0)).await; + assert_eq!((s, &a["shares_held"]), (Status::Ok, &json!([1])), "{a}"); + let (s, a) = post_share_from(&client, "198.51.100.2", h.wrapped(&bundle, &info, 2)).await; + assert_eq!((s, &a["outcome"]), (Status::Ok, &json!("unlocked")), "{a}"); +} + +/// The round-3 security probe, inverted: one planted file whose name or +/// type the mirror refuses used to stop the node booting. +#[tokio::test] +async fn planted_mirror_files_never_stop_a_node_from_booting() { + let dir = tempfile::tempdir().unwrap(); + let h = Holders::new(); + let escrow = json!({"escrow": {"recipients": h.recipients(), "allow_software": true}}); + drop(app(dir.path(), 1, escrow.clone()).await.unwrap()); + let mirror = dir.path().join("at-rest/_tc/at-rest"); + for name in [ + "ark-0000000002-x y.json", + "ark-..json", + "ark-0000000002-x+y.json", + "record-x y.json", + ] { + std::fs::write(mirror.join(name), b"junk").unwrap(); + } + std::fs::create_dir(mirror.join("ark-0000000002-d.json")).unwrap(); + std::os::unix::fs::symlink("/dev/zero", mirror.join("ark-0000000002-z.json")).unwrap(); + + let client = app(dir.path(), 1, escrow).await.unwrap(); + let (_, info) = get_json(&client, "/info").await; + assert_eq!( + info["at_rest"], + json!({"mode":"off","state":"unlocked","encrypting":false}) + ); +} + +#[tokio::test] +async fn misconfigurations_are_refused_at_boot() { + let dir = tempfile::tempdir().unwrap(); + let h = Holders::new(); + // Software holders without the opt-in. + let err = refused( + dir.path(), + 1, + json!({"escrow": {"recipients": h.recipients()}}), + ) + .await; + assert!(err.contains("software keys"), "{err}"); + // Both escrow and escrow_none. + let err = refused( + dir.path(), + 1, + json!({"escrow_none": true, "escrow": {"recipients": h.recipients(), "allow_software": true}}), + ) + .await; + assert!(err.contains("not both"), "{err}"); + // No seal path: encrypt is refused even on a self-hosted Static root + // that acknowledges having no escrow. + let err = refused( + dir.path(), + 1, + json!({"mode": "encrypt", "escrow_none": true}), + ) + .await; + assert!(err.contains("no at-rest seal path"), "{err}"); + // Unknown keys in the section are refused, not ignored. + refused(dir.path(), 1, json!({"escrow_nonee": true})).await; +} diff --git a/tinycloud-node-server/tests/at_rest_env.rs b/tinycloud-node-server/tests/at_rest_env.rs new file mode 100644 index 00000000..b21dd836 --- /dev/null +++ b/tinycloud-node-server/tests/at_rest_env.rs @@ -0,0 +1,46 @@ +//! The compose env lines `tinycloud-at-rest recipients` prints parse into +//! `at_rest` exactly as the node loads them (main.rs's provider chain). +//! One test in its own process, since it sets process environment. + +use rocket::figment::{ + providers::{Env, Serialized}, + Figment, +}; +use tinycloud::config::{AtRestMode, Config}; + +#[test] +fn compose_env_lines_configure_at_rest() { + let recipients = "[{holder=\"sam\",recipient=\"age1yubikey1q2w7u3vpya839jxxuq8g0sedh3d740d4xvn639sqhr95ejj8vu3hyfumptt\"},\ +{holder=\"hunter\",recipient=\"age1qthuv2xtsp2nrz0apnfmj49fkdwp46zygpy4vv8ellvh47203psse695k4\"},\ +{holder=\"safe\",recipient=\"age1tag1qt8lw0ual6avlwmwatk888yqnmdamm7xfd0wak53ut6elz5c4swx2yqdj4e\"}]"; + let fp = "ab".repeat(32); + std::env::set_var("TINYCLOUD_AT_REST__MODE", "encrypt"); + std::env::set_var("TINYCLOUD_AT_REST__ESCROW__RECIPIENTS", recipients); + std::env::set_var("TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP", &fp); + + let config: Config = Figment::from(rocket::Config::default()) + .merge(Serialized::defaults(Config::default())) + .merge(Env::prefixed("TINYCLOUD_").split("_").global()) + .merge(Env::prefixed("TINYCLOUD_").split("__").global()) + .extract() + .expect("at_rest env parses"); + + assert_eq!(config.at_rest.mode, AtRestMode::Encrypt); + let holders: Vec<_> = config + .at_rest + .escrow + .recipients + .iter() + .map(|h| h.holder.as_str()) + .collect(); + assert_eq!(holders, ["sam", "hunter", "safe"]); + assert!(config.at_rest.escrow.recipients[0] + .recipient + .starts_with("age1yubikey1q2w7")); + assert_eq!( + config.at_rest.escrow.verified_fp.as_deref(), + Some(fp.as_str()) + ); + assert!(!config.at_rest.escrow.allow_software); + assert!(!config.at_rest.escrow_none); +}