From 96fcb2fb88b788d607a0557178430e626a882ea3 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 09:59:20 +0000 Subject: [PATCH 1/5] feat(TC-887): seal KV blocks in the FileSystem store; prepare writes before the invoke lock The FileSystem block store follows the at-rest mode and keys: sealed writes under v1/{space_tag}/{object_tag} (in-memory and temp-file staging; atomic rename), reads and ranges through the read plan (sealed, then legacy unless strict; 503 while locked), plaintext sizes by tag, sealed removal. lib.rs links the store to AtRestRuntime; an S3 block store refuses a non-off mode until it can seal. New ImmutableWriteStore::prepare_write hook, which invoke calls for each written space before the SQLite writer lock and transaction; the FS store creates the space's data key there. Invariant: no key-store I/O inside invoke's writer lock or transaction (a DEK insert there deadlocked SQLite nodes on the first sealed put). --- tinycloud-core/src/db.rs | 145 ++++++ tinycloud-core/src/storage/either.rs | 6 + tinycloud-core/src/storage/mod.rs | 10 + tinycloud-node-server/Cargo.toml | 2 + tinycloud-node-server/src/lib.rs | 4 +- .../src/storage/file_system.rs | 416 +++++++++++++++--- .../src/storage/file_system/sealed_tests.rs | 402 +++++++++++++++++ tinycloud-node-server/src/storage/mod.rs | 24 + tinycloud-node-server/src/storage/sealed.rs | 64 ++- .../tests/at_rest_fs_node.rs | 281 ++++++++++++ .../tests/did_resolution_single_entry.rs | 2 +- 11 files changed, 1277 insertions(+), 79 deletions(-) create mode 100644 tinycloud-node-server/src/storage/file_system/sealed_tests.rs create mode 100644 tinycloud-node-server/tests/at_rest_fs_node.rs diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index c2d97193..415077cb 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -2173,6 +2173,18 @@ where // every earlier same-space commit and no same-space writer can commit // until this one does. A failed precondition is a 412, never a false // success (see `postgres_conditional_put_is_atomic_under_concurrency`). + // TC-887: let the block store prepare each written space (e.g. create + // its at-rest data key) before the write lock and transaction below; + // `persist` runs inside them, where such a write would deadlock on + // SQLite. A failure here fails the request before the transaction. + let written_spaces: std::collections::HashSet<&SpaceId> = + stages.keys().map(|(space, _)| space).collect(); + for space in written_spaces { + self.storage + .prepare_write(space) + .await + .map_err(TxStoreError::StoreWrite)?; + } let sequence_spaces = invocation.0.spaces().cloned().collect::>(); let _sequence_guards = self.acquire_space_sequence_guards(&sequence_spaces).await; let _writer = match &self.writer_lock { @@ -5076,6 +5088,139 @@ mod test { self.inner.persist(space, stage).await } } + /// Records whether the SQLite writer lock was free when each write-path + /// hook ran, and can fail `persist`. + #[derive(Clone, Default)] + struct LockProbeStore { + inner: MemoryStore, + lock: Arc>>>, + log: Arc>>, + fail_persist: bool, + } + impl LockProbeStore { + fn record(&self, hook: &'static str) { + let free = self.lock.get().expect("lock installed").try_lock().is_ok(); + self.log.lock().push((hook, free)); + } + } + #[async_trait::async_trait] + impl StorageSetup for LockProbeStore { + type Error = std::io::Error; + async fn create(&self, space: &SpaceId) -> Result<(), Self::Error> { + self.inner.create(space).await + } + } + #[async_trait::async_trait] + impl ImmutableReadStore for LockProbeStore { + type Error = std::io::Error; + type Readable = futures::io::Cursor>; + async fn contains(&self, space: &SpaceId, hash: &Hash) -> Result { + self.inner.contains(space, hash).await + } + async fn read( + &self, + space: &SpaceId, + hash: &Hash, + ) -> Result>, Self::Error> { + self.inner.read(space, hash).await + } + async fn read_range( + &self, + space: &SpaceId, + hash: &Hash, + range: crate::storage::ByteRangeSpec, + ) -> Result>, Self::Error> { + self.inner.read_range(space, hash, range).await + } + } + #[async_trait::async_trait] + impl ImmutableWriteStore for LockProbeStore { + type Error = std::io::Error; + async fn prepare_write(&self, _space: &SpaceId) -> Result<(), Self::Error> { + self.record("prepare_write"); + Ok(()) + } + async fn persist( + &self, + space: &SpaceId, + stage: HashBuffer>, + ) -> Result { + self.record("persist"); + if self.fail_persist { + return Err(std::io::Error::other("injected persist failure")); + } + self.inner.persist(space, stage).await + } + } + + /// TC-887 invariant: a block store's own database writes (e.g. creating a + /// space's at-rest data key) happen in `prepare_write`, which runs before + /// the invocation takes the SQLite writer lock; `persist` runs while it is + /// held. A key-store write in `persist` would deadlock a SQLite node. + #[tokio::test] + async fn prepare_write_runs_before_the_writer_lock_and_persist_inside_it() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + for fail_persist in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory.path().join("probe.sqlite").display() + ); + let storage = LockProbeStore { + fail_persist, + ..Default::default() + }; + let db = SpaceDatabase::new( + Database::connect(url).await.unwrap(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let lock = db + .sqlite_writer_lock() + .expect("a SQLite file node has a writer lock"); + assert!(storage.lock.set(lock).is_ok()); + let space = test_space_id("prepare-write-probe"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&db.conn) + .await + .unwrap(); + let path: Path = "probe/value".parse().unwrap(); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"probe").await.unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + db.invoke_internal_kv_put::( + space.clone(), + path.clone(), + Metadata(Default::default()), + stage, + None, + ), + ) + .await + .expect("kv put must not hang"); + assert_eq!( + *storage.log.lock(), + vec![("prepare_write", true), ("persist", false)], + "prepare_write must see the lock free and persist must run under it" + ); + if fail_persist { + // The transaction rolls back after a successful prepare: no KV + // row, so whatever prepare created stays unused. + assert!(matches!(result, Err(TxStoreError::StoreWrite(_)))); + assert!(db.kv_get(&space, &path).await.unwrap().is_none()); + } else { + assert!(result.is_ok()); + assert!(db.kv_get(&space, &path).await.unwrap().is_some()); + } + } + } + #[tokio::test] async fn legacy_freeze_ack_waits_for_actual_in_flight_kv_storage_and_commit() { use crate::storage::memory::MemoryStaging; diff --git a/tinycloud-core/src/storage/either.rs b/tinycloud-core/src/storage/either.rs index 9d0a0475..0c2020bd 100644 --- a/tinycloud-core/src/storage/either.rs +++ b/tinycloud-core/src/storage/either.rs @@ -150,6 +150,12 @@ where S::Writable: 'static, { type Error = EitherError; + async fn prepare_write(&self, space: &SpaceId) -> Result<(), Self::Error> { + match self { + Self::A(a) => a.prepare_write(space).await.map_err(Self::Error::A), + Self::B(b) => b.prepare_write(space).await.map_err(Self::Error::B), + } + } async fn persist( &self, space: &SpaceId, diff --git a/tinycloud-core/src/storage/mod.rs b/tinycloud-core/src/storage/mod.rs index 9ddebac7..8ca0a1de 100644 --- a/tinycloud-core/src/storage/mod.rs +++ b/tinycloud-core/src/storage/mod.rs @@ -175,6 +175,13 @@ where S::Writable: 'static, { type Error: StdError + Send + Sync; + /// Called for each space an invocation writes, before its database write + /// lock and transaction are taken; `persist` later runs inside them. A + /// store that needs its own database writes for a space (e.g. creating + /// the space's at-rest data key) does them here. Default: nothing. + async fn prepare_write(&self, _space: &SpaceId) -> Result<(), Self::Error> { + Ok(()) + } async fn persist( &self, space: &SpaceId, @@ -263,6 +270,9 @@ where W::Writable: 'static, { type Error = S::Error; + async fn prepare_write(&self, space: &SpaceId) -> Result<(), Self::Error> { + S::prepare_write(self, space).await + } async fn persist( &self, space: &SpaceId, diff --git a/tinycloud-node-server/Cargo.toml b/tinycloud-node-server/Cargo.toml index 0b1a61c0..435a6ed8 100644 --- a/tinycloud-node-server/Cargo.toml +++ b/tinycloud-node-server/Cargo.toml @@ -138,6 +138,8 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd # zlib compressibility of raw stored bytes in tests/at_rest_harness.rs (TC-887). # Already in the lock through ssi-status; adds no new crate. flate2 = "1" +# TC-887: a switchable at-rest state in the sealed block-store tests. +parking_lot = "0.12" k256 = { version = "0.13.4", features = ["ecdsa"] } sha3 = "0.10" # The Rust half of the JS SDK: builds the same host and session SIWE diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 42221362..55a0aadd 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -649,10 +649,12 @@ pub async fn app_with_control( // TC-887: at-rest state, before block storage opens so storage can hold // it; the keyring is installed after migrations (`at_rest::setup`). let at_rest_runtime = at_rest::AtRestRuntime::new(&tinycloud_config.at_rest); + let mut blocks = tinycloud_config.storage.blocks.open().await?; + storage::attach_at_rest(&mut blocks, at_rest_runtime.clone())?; let tinycloud = TinyCloud::new_with_encryption( database_connection, - tinycloud_config.storage.blocks.open().await?, + blocks, key_setup.setup(()).await?, Some(webhook_encryption.clone()), ) diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index 3250857c..842e312d 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -1,4 +1,11 @@ -use super::size::SpaceSizes; +use super::{ + sealed::{ + open_sealed, plan_range, read_plan, seal_file, seal_vec, space_tag_of, write_target, + AtRestHandle, AtRestStoreError, AtRestView, ReadPlan, SealFileError, SealedName, + WriteTarget, + }, + size::SpaceSizes, +}; use core::pin::Pin; use futures::{ future::{Either as AsyncEither, TryFutureExt}, @@ -16,7 +23,11 @@ use std::{ }; use tempfile::{NamedTempFile, PathPersistError}; use tinycloud_auth::{resource::SpaceId, ssi::dids::DIDBuf}; -use tinycloud_core::{at_rest::plaintext_len, hash::Hash, storage::*}; +use tinycloud_core::{ + at_rest::{plaintext_len, DataKeyProvider, DecryptReader, HEADER_LEN}, + hash::Hash, + storage::*, +}; use tokio::{ fs::{create_dir_all, metadata, remove_file, File}, io::{AsyncReadExt, AsyncSeekExt}, @@ -29,8 +40,17 @@ use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; pub struct FileSystemStore { path: PathBuf, sizes: SpaceSizes, + at_rest: Option, +} + +/// A staged block on its way into the store. +enum Staged { + File(TempFileStage), + Bytes(Vec), } +type PlainReader = Compat>; + impl FileSystemStore { async fn new(path: PathBuf) -> Result { // get the size of the directory @@ -38,7 +58,21 @@ impl FileSystemStore { store_sizes(&path).await?, sealed_sizes(&path.join(SEALED_DIR)).await?, ); - Ok(Self { path, sizes }) + Ok(Self { + path, + sizes, + at_rest: None, + }) + } + + /// Links the store to the node's at-rest state (TC-887). Without it the + /// store behaves as at-rest mode `off`. + pub fn attach_at_rest(&mut self, handle: AtRestHandle) { + self.at_rest = Some(handle); + } + + fn view(&self) -> AtRestView { + AtRestView::of(self.at_rest.as_ref()) } fn get_path(&self, space: &SpaceId, mh: &Hash) -> PathBuf { @@ -48,14 +82,264 @@ impl FileSystemStore { .join(base64::encode_config(mh.as_ref(), base64::URL_SAFE)) } - async fn increment_size(&self, space: &SpaceId, size: u64) { - self.sizes.increment_size(space, None, size).await; + fn sealed_path(&self, name: &SealedName) -> PathBuf { + self.path + .join(SEALED_DIR) + .join(&name.space_tag) + .join(&name.object_tag) } - async fn decrement_size(&self, space: &SpaceId, size: u64) { - self.sizes.decrement_size(space, None, size).await; + + async fn persist_staged( + &self, + space: &SpaceId, + hash: &Hash, + staged: Staged, + ) -> Result<(), FileSystemStoreError> { + match write_target(&self.view(), space, hash).await? { + WriteTarget::Legacy => { + let path = self.get_path(space, hash); + if path.exists() { + return Ok(()); + } + let size = match staged { + Staged::File(t_file) => { + let size = t_file.size().await?; + let (_, tmp) = t_file.into_inner(); + tmp.persist(path)?; + size + } + Staged::Bytes(v) => { + let file = File::create(path).await?; + let size = v.len() as u64; + let mut writer = futures::io::BufWriter::new(file.compat()); + writer.write_all(&v).await?; + writer.flush().await?; + size + } + }; + self.sizes.increment_size(space, None, size).await; + } + WriteTarget::Sealed { name, key } => { + // Only the sealed name dedups: a legacy copy never suppresses + // the sealed write. + let dest = self.sealed_path(&name); + if dest.exists() { + return Ok(()); + } + let dir = dest.parent().expect("sealed path has a parent"); + create_dir_all(dir).await?; + let (size, sealed) = match staged { + Staged::File(t_file) => { + let size = t_file.size().await?; + let (_, tmp) = t_file.into_inner(); + let sealed = seal_file(&key, space, hash, &tmp, size) + .await + .map_err(FileSystemStoreError::from)?; + (size, sealed) + } + Staged::Bytes(v) => { + let size = v.len() as u64; + let ct = seal_vec(key, space, hash, v).await?; + let mut out = NamedTempFile::new_in(dir)?; + std::io::Write::write_all(&mut out, &ct)?; + (size, out) + } + }; + sealed.persist(&dest).map_err(|e| e.error)?; + self.sizes + .increment_size(space, Some(&name.space_tag), size) + .await; + } + } + Ok(()) + } + + async fn open_legacy( + &self, + space: &SpaceId, + id: &Hash, + ) -> Result>, FileSystemStoreError> { + match File::open(self.get_path(space, id)).await { + Ok(file) => { + let size = file.metadata().await?.len(); + Ok(Some(Content::new(size, file.take(size).compat()))) + } + Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// Opens the block's sealed file, or `None` if it is absent. + async fn open_sealed_file( + &self, + name: &SealedName, + provider: &dyn DataKeyProvider, + space: &SpaceId, + id: &Hash, + ) -> Result>, FileSystemStoreError> { + let file = match File::open(self.sealed_path(name)).await { + Ok(file) => file, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + let len = file.metadata().await?.len(); + let (size, reader) = open_sealed(provider, space, id, file.take(len).compat(), len) + .await? + .into_inner(); + Ok(Some(Content::new(size, AsyncEither::Right(reader)))) + } + + async fn read_block( + &self, + space: &SpaceId, + id: &Hash, + ) -> Result>, FileSystemStoreError> { + let legacy = |c: Option>| { + c.map(|c| { + let (size, r) = c.into_inner(); + Content::new(size, AsyncEither::Left(r)) + }) + }; + match read_plan(&self.view(), space, id)? { + ReadPlan::Legacy => Ok(legacy(self.open_legacy(space, id).await?)), + ReadPlan::LegacyWhileLocked => match legacy(self.open_legacy(space, id).await?) { + Some(c) => Ok(Some(c)), + None => Err(AtRestStoreError::Unavailable("at-rest keys are locked").into()), + }, + ReadPlan::SealedThenLegacy { name, provider } => { + match self.open_sealed_file(&name, &*provider, space, id).await? { + Some(c) => Ok(Some(c)), + None => Ok(legacy(self.open_legacy(space, id).await?)), + } + } + ReadPlan::Sealed { name, provider } => { + match self.open_sealed_file(&name, &*provider, space, id).await? { + Some(c) => Ok(Some(c)), + None if self.get_path(space, id).exists() => { + Err(AtRestStoreError::LegacyRefused.into()) + } + None => Ok(None), + } + } + } + } + + async fn range_legacy( + &self, + space: &SpaceId, + id: &Hash, + range: ByteRangeSpec, + ) -> Result>, FileSystemStoreError> { + match File::open(self.get_path(space, id)).await { + Ok(mut file) => { + let total_size = file.metadata().await?.len(); + let Some(range) = range.resolve(total_size) else { + return Ok(Some(RangeRead::Unsatisfiable { total_size })); + }; + file.seek(SeekFrom::Start(range.start())).await?; + let content = Content::new( + range.len(), + AsyncEither::Left(file.take(range.len()).compat()), + ); + Ok(Some(RangeRead::Content { + total_size, + range, + content, + })) + } + Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// A range of the block's sealed file, or `None` if it is absent. Totals + /// and the 416 decision use the plaintext length. + async fn range_sealed( + &self, + name: &SealedName, + provider: &dyn DataKeyProvider, + space: &SpaceId, + id: &Hash, + spec: ByteRangeSpec, + ) -> Result>, FileSystemStoreError> { + let mut file = match File::open(self.sealed_path(name)).await { + Ok(file) => file, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + let len = file.metadata().await?.len(); + let mut header = [0u8; HEADER_LEN]; + file.read_exact(&mut header) + .await + .map_err(|_| AtRestStoreError::Integrity("object shorter than its header"))?; + let plan = plan_range(provider, space, &header, len, spec).await?; + let Some(r) = plan.range else { + return Ok(Some(RangeRead::Unsatisfiable { + total_size: plan.plaintext_len(), + })); + }; + file.seek(SeekFrom::Start(r.ciphertext_start())).await?; + let span = r.ciphertext_end() - r.ciphertext_start() + 1; + let reader = plan.reader(space, id, r, file.take(span).compat())?; + let range = r.plaintext_range(); + Ok(Some(RangeRead::Content { + total_size: plan.plaintext_len(), + range, + content: Content::new(range.len(), AsyncEither::Right(reader)), + })) + } + + async fn range_block( + &self, + space: &SpaceId, + id: &Hash, + spec: ByteRangeSpec, + ) -> Result>, FileSystemStoreError> { + match read_plan(&self.view(), space, id)? { + ReadPlan::Legacy => self.range_legacy(space, id, spec).await, + ReadPlan::LegacyWhileLocked => match self.range_legacy(space, id, spec).await? { + Some(r) => Ok(Some(r)), + None => Err(AtRestStoreError::Unavailable("at-rest keys are locked").into()), + }, + ReadPlan::SealedThenLegacy { name, provider } => { + match self + .range_sealed(&name, &*provider, space, id, spec) + .await? + { + Some(r) => Ok(Some(r)), + None => self.range_legacy(space, id, spec).await, + } + } + ReadPlan::Sealed { name, provider } => { + match self + .range_sealed(&name, &*provider, space, id, spec) + .await? + { + Some(r) => Ok(Some(r)), + None if self.get_path(space, id).exists() => { + Err(AtRestStoreError::LegacyRefused.into()) + } + None => Ok(None), + } + } + } + } + + /// Whether the block exists under a name this node may read. + fn exists(&self, space: &SpaceId, id: &Hash) -> Result { + Ok(match read_plan(&self.view(), space, id)? { + ReadPlan::Legacy | ReadPlan::LegacyWhileLocked => self.get_path(space, id).exists(), + ReadPlan::SealedThenLegacy { name, .. } => { + self.sealed_path(&name).exists() || self.get_path(space, id).exists() + } + ReadPlan::Sealed { name, .. } => self.sealed_path(&name).exists(), + }) } } +/// Plaintext from a legacy file, or authenticated plaintext from a sealed one. +pub type FsReadable = AsyncEither>; + #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct FileSystemConfig { path: PathBuf, @@ -114,12 +398,21 @@ pub enum FileSystemStoreError { AtRest(#[from] super::sealed::AtRestStoreError), } +impl From for FileSystemStoreError { + fn from(e: SealFileError) -> Self { + match e { + SealFileError::AtRest(e) => Self::AtRest(e), + SealFileError::Io(e) => Self::Io(e), + } + } +} + #[async_trait] impl ImmutableReadStore for FileSystemStore { type Error = FileSystemStoreError; - type Readable = Compat>; + type Readable = FsReadable; async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { - Ok(self.get_path(space, id).exists()) + self.exists(space, id) } async fn read( &self, @@ -127,14 +420,7 @@ impl ImmutableReadStore for FileSystemStore { id: &Hash, ) -> Result>, Self::Error> { let start = Instant::now(); - let result = match File::open(self.get_path(space, id)).await { - Ok(file) => { - let size = file.metadata().await?.len(); - Ok(Some(Content::new(size, file.take(size).compat()))) - } - Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), - Err(e) => Err(e.into()), - }; + let result = self.read_block(space, id).await; crate::prometheus::observe_stage( crate::prometheus::InvocationStage::BlockRead, crate::prometheus::StageOutcome::from(result.is_ok()), @@ -149,23 +435,7 @@ impl ImmutableReadStore for FileSystemStore { id: &Hash, range: ByteRangeSpec, ) -> Result>, Self::Error> { - match File::open(self.get_path(space, id)).await { - Ok(mut file) => { - let total_size = file.metadata().await?.len(); - let Some(range) = range.resolve(total_size) else { - return Ok(Some(RangeRead::Unsatisfiable { total_size })); - }; - file.seek(SeekFrom::Start(range.start())).await?; - let content = Content::new(range.len(), file.take(range.len()).compat()); - Ok(Some(RangeRead::Content { - total_size, - range, - content, - })) - } - Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), - Err(e) => Err(e.into()), - } + self.range_block(space, id, range).await } } @@ -173,7 +443,11 @@ impl ImmutableReadStore for FileSystemStore { impl StoreSize for FileSystemStore { type Error = FileSystemStoreError; async fn total_size(&self, space: &SpaceId) -> Result, Self::Error> { - Ok(self.sizes.get_size(space, None).await) + let view = self.view(); + Ok(self + .sizes + .get_size(space, space_tag_of(&view, space).as_deref()) + .await) } } @@ -313,6 +587,9 @@ impl ImmutableStaging for TempFileSystemStage { #[async_trait] impl ImmutableWriteStore for FileSystemStore { type Error = FileSystemStoreError; + async fn prepare_write(&self, space: &SpaceId) -> Result<(), Self::Error> { + Ok(super::sealed::prepare_write(&self.view(), space).await?) + } async fn persist( &self, space: &SpaceId, @@ -322,12 +599,7 @@ impl ImmutableWriteStore for FileSystemStore { let result = async move { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - if !self.contains(space, &hash).await? { - let size = f.size().await?; - let (_, path) = f.into_inner(); - path.persist(self.get_path(space, &hash))?; - self.increment_size(space, size).await; - } + self.persist_staged(space, &hash, Staged::File(f)).await?; Ok(hash) } .await; @@ -343,6 +615,9 @@ impl ImmutableWriteStore for FileSystemStore { #[async_trait] impl ImmutableWriteStore for FileSystemStore { type Error = FileSystemStoreError; + async fn prepare_write(&self, space: &SpaceId) -> Result<(), Self::Error> { + Ok(super::sealed::prepare_write(&self.view(), space).await?) + } async fn persist( &self, space: &SpaceId, @@ -352,14 +627,7 @@ impl ImmutableWriteStore for FileSystemStore { let result = async move { let (mut h, v) = staged.into_inner(); let hash = h.finalize(); - if !self.contains(space, &hash).await? { - let file = File::create(self.get_path(space, &hash)).await?; - let size = v.len() as u64; - let mut writer = futures::io::BufWriter::new(file.compat()); - writer.write_all(&v).await?; - writer.flush().await?; - self.increment_size(space, size).await; - } + self.persist_staged(space, &hash, Staged::Bytes(v)).await?; Ok(hash) } .await; @@ -377,6 +645,9 @@ impl ImmutableWriteStore Result<(), Self::Error> { + Ok(super::sealed::prepare_write(&self.view(), space).await?) + } async fn persist( &self, space: &SpaceId, @@ -386,25 +657,11 @@ impl ImmutableWriteStore { - let size = t_file.size().await?; - let (_, path) = t_file.into_inner(); - path.persist(self.get_path(space, &hash))?; - self.increment_size(space, size).await; - } - AsyncEither::Right(v) => { - let file = File::create(self.get_path(space, &hash)).await?; - let size = v.len() as u64; - let mut writer = futures::io::BufWriter::new(file.compat()); - writer.write_all(&v).await?; - writer.flush().await?; - self.increment_size(space, size).await; - } - } + let staged = match f { + AsyncEither::Left(t_file) => Staged::File(t_file), + AsyncEither::Right(v) => Staged::Bytes(v), }; + self.persist_staged(space, &hash, staged).await?; Ok(hash) } .await; @@ -421,6 +678,26 @@ impl ImmutableWriteStore Result, Self::Error> { + // A sealed copy first (if keys can name it), then the legacy file. + let view = self.view(); + let sealed = view + .object_name_key() + .map(|nk| SealedName::new(&nk, space, id)); + if let Some(name) = &sealed { + let path = self.sealed_path(name); + match metadata(&path).await { + Ok(m) => { + remove_file(path).await?; + let size = plaintext_len(m.len()).unwrap_or(0); + self.sizes + .decrement_size(space, Some(&name.space_tag), size) + .await; + return Ok(Some(())); + } + Err(e) if e.kind() == ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + } let path = self.get_path(space, id); let size = match metadata(&path).await { Ok(m) => m.len(), @@ -429,7 +706,7 @@ impl ImmutableDeleteStore for FileSystemStore { }; match remove_file(path).await { Ok(()) => { - self.decrement_size(space, size).await; + self.sizes.decrement_size(space, None, size).await; Ok(Some(())) } Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), @@ -446,6 +723,9 @@ impl StorageConfig for TempFileSystemStage { } } +#[cfg(test)] +mod sealed_tests; + #[cfg(test)] mod test { use super::*; diff --git a/tinycloud-node-server/src/storage/file_system/sealed_tests.rs b/tinycloud-node-server/src/storage/file_system/sealed_tests.rs new file mode 100644 index 00000000..a36c231e --- /dev/null +++ b/tinycloud-node-server/src/storage/file_system/sealed_tests.rs @@ -0,0 +1,402 @@ +//! TC-887 S3b: the FileSystem block store with encryption at rest. + +use super::*; +use crate::config::AtRestMode; +use crate::storage::sealed::{AtRestSource, AtRestView}; +use futures::io::AsyncReadExt as _; +use parking_lot::RwLock; +use rocket::http::Status; +use std::sync::Arc; +use tinycloud_core::at_rest::{ciphertext_len, DekId, FixedKeyProvider, SEGMENT_LEN}; + +const SEG: usize = SEGMENT_LEN; +const MARKER: &[u8] = b"TC887-FS-MARKER-plaintext-must-not-reach-disk"; + +/// An at-rest state the test can switch, as a recovery or restart would. +struct Switch(RwLock); + +impl AtRestSource for Switch { + fn view(&self) -> AtRestView { + self.0.read().clone() + } +} + +impl Switch { + fn set(&self, v: AtRestView) { + *self.0.write() = v; + } +} + +fn keys() -> FixedKeyProvider { + FixedKeyProvider::new([9; 32]) +} + +fn view(mode: AtRestMode, encrypting: bool, keys: Option) -> AtRestView { + AtRestView { + mode, + encrypting, + provider: keys.map(|k| Arc::new(k) as Arc), + } +} + +fn encrypt() -> AtRestView { + view(AtRestMode::Encrypt, true, Some(keys())) +} + +fn space() -> SpaceId { + "tinycloud:key:test:default".parse().unwrap() +} + +async fn open_store(dir: &Path, v: AtRestView) -> (FileSystemStore, Arc) { + let mut store = FileSystemConfig::new(dir).open().await.unwrap(); + let switch = Arc::new(Switch(RwLock::new(v))); + store.attach_at_rest(AtRestHandle::new(switch.clone())); + store.create(&space()).await.unwrap(); + (store, switch) +} + +fn value(n: usize) -> Vec { + let mut v: Vec = (0..n).map(|i| (i * 7 % 251) as u8).collect(); + if n >= MARKER.len() { + v[..MARKER.len()].copy_from_slice(MARKER); + } + v +} + +async fn put_bytes(store: &FileSystemStore, data: &[u8]) -> Result { + let mut staged = HashBuffer::new(Vec::new()); + staged.write_all(data).await.unwrap(); + ImmutableWriteStore::::persist(store, &space(), staged).await +} + +async fn put_file(store: &FileSystemStore, data: &[u8]) -> Result { + let mut staged = TempFileSystemStage.stage(&space()).await.unwrap(); + staged.write_all(data).await.unwrap(); + staged.flush().await.unwrap(); + ImmutableWriteStore::::persist(store, &space(), staged).await +} + +async fn read_all(store: &FileSystemStore, h: &Hash) -> Result>, String> { + let Some(content) = store.read(&space(), h).await.map_err(|e| e.to_string())? else { + return Ok(None); + }; + let len = content.len(); + let mut out = Vec::new(); + let mut r = Box::pin(content); + r.read_to_end(&mut out).await.map_err(|e| e.to_string())?; + assert_eq!( + out.len() as u64, + len, + "Content::len is the plaintext length" + ); + Ok(Some(out)) +} + +fn sealed_files(dir: &Path) -> Vec { + let mut out = Vec::new(); + let Ok(tags) = std::fs::read_dir(dir.join(SEALED_DIR)) else { + return out; + }; + for tag in tags.flatten() { + for obj in std::fs::read_dir(tag.path()).unwrap().flatten() { + out.push(obj.path()); + } + } + out.sort(); + out +} + +fn status(e: &FileSystemStoreError) -> Option { + match e { + FileSystemStoreError::AtRest(e) => Some(e.status()), + _ => None, + } +} + +#[tokio::test] +async fn encrypt_mode_seals_both_staging_paths() { + for (n, via_file) in [ + (10, false), + (SEG + 5, false), + (3 * SEG, true), + (SEG - 1, true), + (0, true), + ] { + let dir = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), encrypt()).await; + let data = value(n); + let h = if via_file { + put_file(&store, &data).await.unwrap() + } else { + put_bytes(&store, &data).await.unwrap() + }; + assert!(!store.get_path(&space(), &h).exists(), "no legacy copy"); + let files = sealed_files(dir.path()); + assert_eq!(files.len(), 1); + let raw = std::fs::read(&files[0]).unwrap(); + assert_eq!(raw.len() as u64, ciphertext_len(n as u64).unwrap()); + assert!( + !raw.windows(MARKER.len()).any(|w| w == MARKER), + "plaintext on disk" + ); + let rel = files[0] + .strip_prefix(dir.path()) + .unwrap() + .to_string_lossy() + .into_owned(); + assert!( + !rel.contains("tinycloud") && !rel.contains("key:test"), + "{rel}" + ); + assert_eq!(read_all(&store, &h).await.unwrap().unwrap(), data); + assert!(store.contains(&space(), &h).await.unwrap()); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(n as u64)); + } +} + +#[tokio::test] +async fn sealed_ranges_use_plaintext_offsets_and_totals() { + let dir = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), encrypt()).await; + let n = 3 * SEG + 9; + let data = value(n); + let h = put_bytes(&store, &data).await.unwrap(); + let edges = [0, 1, SEG - 1, SEG, SEG + 1, 2 * SEG, n - 1]; + for &a in &edges { + for &b in edges.iter().filter(|&&b| b >= a) { + let spec = ByteRangeSpec::Inclusive { + start: a as u64, + end: b as u64, + }; + match store.read_range(&space(), &h, spec).await.unwrap().unwrap() { + RangeRead::Content { + total_size, + range, + content, + } => { + assert_eq!(total_size, n as u64); + assert_eq!((range.start(), range.end()), (a as u64, b as u64)); + assert_eq!(content.len(), (b - a + 1) as u64); + let mut out = Vec::new(); + Box::pin(content).read_to_end(&mut out).await.unwrap(); + assert_eq!(out, data[a..=b], "{a}..={b}"); + } + RangeRead::Unsatisfiable { .. } => panic!("{a}..={b} unsatisfiable"), + } + } + } + assert!(matches!( + store + .read_range(&space(), &h, ByteRangeSpec::From { start: n as u64 }) + .await + .unwrap() + .unwrap(), + RangeRead::Unsatisfiable { total_size } if total_size == n as u64 + )); +} + +#[tokio::test] +async fn off_mode_writes_legacy_even_with_keys() { + let dir = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), view(AtRestMode::Off, true, Some(keys()))).await; + let data = value(100); + let h = put_bytes(&store, &data).await.unwrap(); + assert!(store.get_path(&space(), &h).exists()); + assert!(sealed_files(dir.path()).is_empty()); + assert_eq!(read_all(&store, &h).await.unwrap().unwrap(), data); +} + +#[tokio::test] +async fn legacy_reads_in_encrypt_and_is_refused_in_strict() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), AtRestView::off()).await; + let data = value(SEG + 1); + let h = put_bytes(&store, &data).await.unwrap(); + + switch.set(encrypt()); + assert_eq!(read_all(&store, &h).await.unwrap().unwrap(), data); + assert!(matches!( + store + .read_range(&space(), &h, ByteRangeSpec::Suffix { length: 3 }) + .await + .unwrap(), + Some(RangeRead::Content { .. }) + )); + + switch.set(view(AtRestMode::Strict, true, Some(keys()))); + for e in [ + store.read(&space(), &h).await.map(|_| ()).unwrap_err(), + store + .read_range(&space(), &h, ByteRangeSpec::From { start: 0 }) + .await + .map(|_| ()) + .unwrap_err(), + ] { + assert!( + matches!( + e, + FileSystemStoreError::AtRest(AtRestStoreError::LegacyRefused) + ), + "{e}" + ); + assert_eq!(status(&e), Some(Status::InternalServerError)); + } + assert!(!store.contains(&space(), &h).await.unwrap()); +} + +/// A legacy copy never suppresses the sealed write in encrypt mode. +#[tokio::test] +async fn a_legacy_copy_does_not_suppress_the_sealed_write() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), AtRestView::off()).await; + let data = value(500); + let h = put_bytes(&store, &data).await.unwrap(); + switch.set(encrypt()); + assert_eq!(put_file(&store, &data).await.unwrap(), h); + assert!(store.get_path(&space(), &h).exists()); + assert_eq!(sealed_files(dir.path()).len(), 1); + // Reads now come from the sealed copy: removing the legacy file changes nothing. + std::fs::remove_file(store.get_path(&space(), &h)).unwrap(); + assert_eq!(read_all(&store, &h).await.unwrap().unwrap(), data); +} + +#[tokio::test] +async fn locked_keys_refuse_writes_and_sealed_reads_but_serve_legacy() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), AtRestView::off()).await; + let legacy = value(64); + let hl = put_bytes(&store, &legacy).await.unwrap(); + switch.set(encrypt()); + let sealed = value(65); + let hs = put_bytes(&store, &sealed).await.unwrap(); + + for locked in [ + view(AtRestMode::Encrypt, false, Some(keys().locked())), + view(AtRestMode::Encrypt, true, Some(keys().locked())), + view(AtRestMode::Encrypt, false, Some(keys())), + ] { + switch.set(locked); + let e = put_bytes(&store, &value(66)).await.unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); + } + switch.set(view(AtRestMode::Encrypt, false, Some(keys().locked()))); + assert_eq!(read_all(&store, &hl).await.unwrap().unwrap(), legacy); + let e = store.read(&space(), &hs).await.map(|_| ()).unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); + let e = store + .read_range(&space(), &hs, ByteRangeSpec::From { start: 0 }) + .await + .map(|_| ()) + .unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); +} + +/// After a restart, sealed bytes are counted by tag; a locked node reports +/// legacy bytes only, and the first quota check after unlocking counts both. +#[tokio::test] +async fn quota_after_unlock_counts_legacy_and_sealed() { + let dir = tempfile::tempdir().unwrap(); + { + let (store, switch) = open_store(dir.path(), AtRestView::off()).await; + put_bytes(&store, &value(1000)).await.unwrap(); + switch.set(encrypt()); + put_bytes(&store, &value(70_000)).await.unwrap(); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(71_000)); + } + // Restart: the boot scan sees 1000 legacy bytes and 70 000 sealed bytes + // under a tag. No `create` (it would reset the legacy entry, as before). + let switch = Arc::new(Switch(RwLock::new(view( + AtRestMode::Encrypt, + false, + Some(keys().locked()), + )))); + let mut store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + store.attach_at_rest(AtRestHandle::new(switch.clone())); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(1000)); + switch.set(encrypt()); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(71_000)); + put_bytes(&store, &value(5)).await.unwrap(); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(71_005)); +} + +#[tokio::test] +async fn tampered_swapped_or_unsupported_sealed_files_fail_closed() { + let dir = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), encrypt()).await; + let (a, b) = ( + value(SEG + 3), + value(SEG + 3).into_iter().rev().collect::>(), + ); + let ha = put_bytes(&store, &a).await.unwrap(); + let hb = put_bytes(&store, &b).await.unwrap(); + let nk = keys().object_name_key().unwrap(); + let pa = store.sealed_path(&SealedName::new(&nk, &space(), &ha)); + let pb = store.sealed_path(&SealedName::new(&nk, &space(), &hb)); + + // Swap the two objects (same space, same size): each name now holds the + // other blob, bound to the other hash. + let (ra, rb) = (std::fs::read(&pa).unwrap(), std::fs::read(&pb).unwrap()); + std::fs::write(&pa, &rb).unwrap(); + std::fs::write(&pb, &ra).unwrap(); + for h in [&ha, &hb] { + let e = read_all(&store, h).await.unwrap_err(); + assert!(e.to_lowercase().contains("integrity"), "{e}"); + } + + // A flipped byte in the second segment: the first segment is served, then the stream fails. + let mut t = ra.clone(); + t[64 + SEG + 16 + 1] ^= 1; + std::fs::write(&pa, &t).unwrap(); + let content = store.read(&space(), &ha).await.unwrap().unwrap(); + let mut out = Vec::new(); + let err = Box::pin(content).read_to_end(&mut out).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + out.len() <= SEG, + "only the authenticated segment was released" + ); + + // A later format generation: a hard error, never legacy. + let mut v2 = ra.clone(); + v2[3] = b'2'; + v2[4] = 2; + std::fs::write(&pa, &v2).unwrap(); + let e = store.read(&space(), &ha).await.map(|_| ()).unwrap_err(); + assert!( + matches!( + e, + FileSystemStoreError::AtRest(AtRestStoreError::Unsupported(2)) + ), + "{e}" + ); + assert_eq!(status(&e), Some(Status::InternalServerError)); +} + +#[tokio::test] +async fn removing_a_sealed_block_drops_its_bytes() { + let dir = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), encrypt()).await; + let h = put_bytes(&store, &value(4096)).await.unwrap(); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(4096)); + assert_eq!(store.remove(&space(), &h).await.unwrap(), Some(())); + assert!(sealed_files(dir.path()).is_empty()); + assert_eq!(store.total_size(&space()).await.unwrap(), Some(0)); + assert_eq!(store.read(&space(), &h).await.unwrap().map(|_| ()), None); +} + +#[tokio::test] +async fn the_current_key_rotates_without_breaking_old_blocks() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), encrypt()).await; + let old = value(300); + let h_old = put_bytes(&store, &old).await.unwrap(); + switch.set(view( + AtRestMode::Encrypt, + true, + Some(FixedKeyProvider::new([9; 32]).with_current(DekId::new(2))), + )); + let new = value(301); + let h_new = put_bytes(&store, &new).await.unwrap(); + assert_eq!(read_all(&store, &h_old).await.unwrap().unwrap(), old); + assert_eq!(read_all(&store, &h_new).await.unwrap().unwrap(), new); +} diff --git a/tinycloud-node-server/src/storage/mod.rs b/tinycloud-node-server/src/storage/mod.rs index e0cad057..a10fa30c 100644 --- a/tinycloud-node-server/src/storage/mod.rs +++ b/tinycloud-node-server/src/storage/mod.rs @@ -3,3 +3,27 @@ pub mod file_system; pub mod s3; pub mod sealed; pub mod size; + +use std::sync::Arc; + +use crate::{at_rest::AtRestRuntime, config::AtRestMode, BlockStores}; +use tinycloud_core::storage::either::Either; + +/// Links the opened block store to the node's at-rest state (TC-887). +/// +/// The FileSystem store seals KV blocks when the mode and keys allow. The S3 +/// store does not seal yet, so a non-`off` mode with S3 storage refuses to +/// boot rather than write plaintext. +pub fn attach_at_rest(stores: &mut BlockStores, runtime: Arc) -> anyhow::Result<()> { + match stores { + Either::B(fs) => { + fs.attach_at_rest(sealed::AtRestHandle::new(runtime)); + Ok(()) + } + Either::A(_) if runtime.mode() == AtRestMode::Off => Ok(()), + Either::A(_) => anyhow::bail!( + "at_rest.mode = \"{}\" is not supported with S3 block storage in this build", + runtime.mode_name() + ), + } +} diff --git a/tinycloud-node-server/src/storage/sealed.rs b/tinycloud-node-server/src/storage/sealed.rs index c289cd6a..67cefd98 100644 --- a/tinycloud-node-server/src/storage/sealed.rs +++ b/tinycloud-node-server/src/storage/sealed.rs @@ -119,15 +119,9 @@ impl AtRestView { } } - pub fn of(runtime: Option<&AtRestRuntime>) -> Self { - match runtime { - None => Self::off(), - Some(rt) => Self { - mode: rt.mode(), - encrypting: rt.encrypting(), - provider: rt.provider(), - }, - } + /// The state a store sees: `off` without a handle, else the handle's. + pub fn of(handle: Option<&AtRestHandle>) -> Self { + handle.map_or_else(Self::off, |h| h.0.view()) } /// The object-name key, if keys are set up and unlocked. @@ -136,6 +130,39 @@ impl AtRestView { } } +/// Where a block store reads its at-rest state from, per operation: the +/// node's [`AtRestRuntime`], whose keys arrive after migrations and may unlock +/// later, or a test double. +pub trait AtRestSource: Send + Sync { + fn view(&self) -> AtRestView; +} + +impl AtRestSource for AtRestRuntime { + fn view(&self) -> AtRestView { + AtRestView { + mode: self.mode(), + encrypting: self.encrypting(), + provider: self.provider(), + } + } +} + +/// A block store's link to the at-rest state. +#[derive(Clone)] +pub struct AtRestHandle(Arc); + +impl AtRestHandle { + pub fn new(source: Arc) -> Self { + Self(source) + } +} + +impl std::fmt::Debug for AtRestHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AtRestHandle").finish_non_exhaustive() + } +} + /// The sealed name of a block: backends lay it out as `v1/{space_tag}/{object_tag}`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SealedName { @@ -171,6 +198,25 @@ pub fn space_tag_of(view: &AtRestView, space: &SpaceId) -> Option { .map(|nk| space_tag(&nk, &space.to_string())) } +/// Gets a space ready for sealed writes before the invocation's database +/// write lock and transaction: creates (or loads) the space's current data +/// key, so `persist` inside the transaction never writes key rows. Errors +/// fail the request before the transaction (503 while keys are locked). +pub async fn prepare_write(view: &AtRestView, space: &SpaceId) -> Result<(), AtRestStoreError> { + if view.mode == AtRestMode::Off { + return Ok(()); + } + if !view.encrypting { + return Err(AtRestStoreError::Unavailable("encryption is not enabled")); + } + let provider = view + .provider + .as_ref() + .ok_or(AtRestStoreError::Unavailable("at-rest keys are not set up"))?; + provider.current_key(&space.to_string()).await?; + Ok(()) +} + /// Where a new block is written. pub enum WriteTarget { Legacy, diff --git a/tinycloud-node-server/tests/at_rest_fs_node.rs b/tinycloud-node-server/tests/at_rest_fs_node.rs new file mode 100644 index 00000000..093cb457 --- /dev/null +++ b/tinycloud-node-server/tests/at_rest_fs_node.rs @@ -0,0 +1,281 @@ +//! TC-887 S3b through the real app: a node on filesystem blocks with +//! `at_rest.mode = "encrypt"` (self-host `escrow_none`) stores KV values sealed +//! under anonymous names and serves them back as plaintext, across a restart. +//! An `off` node on the same code stores them as before. + +use std::path::{Path as FsPath, PathBuf}; + +use anyhow::{anyhow, ensure, Context, Result}; +use k256::ecdsa::SigningKey; +use rocket::{ + figment::{providers::Serialized, Figment}, + http::{ContentType, Header, Status}, + local::asynchronous::Client, +}; +use serde_json::{json, Value}; +use sha3::{Digest, Keccak256}; +use tinycloud_auth::{ + cacaos::{ + siwe::{encode_eip55, Message}, + siwe_cacao::Signature, + }, + resource::{Path, Service, SpaceId}, + siwe_recap::Ability, + ssi::jwk::JWK, +}; +use tinycloud_sdk_wasm::{ + host::{generate_host_siwe_message, siwe_to_delegation_headers, SignedMessage}, + session::{complete_session_setup, prepare_session, Session, SignedSession}, +}; + +const MARKER: &str = "TC887-S3B-NODE-MARKER"; + +async fn node(dir: &FsPath, at_rest: Value) -> Result { + let mut config = tinycloud::config::Config::default(); + config.storage.datadir = dir.into(); + config.storage.database = Some(format!("sqlite:{}?mode=rwc", dir.join("caps.db").display())); + config.keys = serde_json::from_value(json!({ + "type": "static", + "secret": base64::encode_config([113; 32], base64::URL_SAFE_NO_PAD), + }))?; + config.at_rest = serde_json::from_value(at_rest)?; + let figment = Figment::from(rocket::Config { + log_level: rocket::config::LogLevel::Off, + ..rocket::Config::default() + }) + .merge(Serialized::defaults(&config)); + Ok(Client::tracked(tinycloud::app(&figment).await?).await?) +} + +fn wallet() -> Result<(SigningKey, [u8; 20], SpaceId)> { + let key = SigningKey::from_bytes(&[113; 32].into())?; + let public = key.verifying_key().to_encoded_point(false); + let digest = Keccak256::digest(&public.as_bytes()[1..]); + let address: [u8; 20] = digest[12..].try_into()?; + let space = format!( + "tinycloud:pkh:eip155:1:0x{}:at-rest-fs", + encode_eip55(&address) + ) + .parse()?; + Ok((key, address, space)) +} + +fn sign(key: &SigningKey, message: &Message) -> Result<[u8; 65]> { + let (signature, recovery) = + key.sign_prehash_recoverable(&message.eip191_hash().map_err(|e| anyhow!("{e}"))?)?; + let mut bytes = [0; 65]; + bytes[..64].copy_from_slice(&signature.to_bytes()); + bytes[64] = recovery.to_byte() + 27; + Ok(bytes) +} + +fn rfc3339(t: time::OffsetDateTime) -> Result { + Ok(t.format(&time::format_description::well_known::Rfc3339)?) +} + +async fn delegate(client: &Client, headers: &impl serde::Serialize) -> Result<()> { + let auth = serde_json::to_value(headers)?["Authorization"] + .as_str() + .map(str::to_owned) + .context("Authorization header")?; + let response = client + .post("/delegate") + .header(Header::new("Authorization", auth)) + .dispatch() + .await; + let status = response.status(); + ensure!( + status == Status::Ok, + "delegate: {status} {:?}", + response.into_string().await + ); + Ok(()) +} + +/// Host delegation for the space, then a KV session. +async fn session(client: &Client) -> Result { + let (wallet, address, space) = wallet()?; + let now = time::OffsetDateTime::now_utc(); + let peer = client + .get(format!("/peer/generate/{space}")) + .dispatch() + .await; + ensure!(peer.status() == Status::Ok, "peer generation"); + let host = generate_host_siwe_message(serde_json::from_value(json!({ + "address": format!("0x{}", encode_eip55(&address)), + "chainId": 1, "domain": "localhost", + "issuedAt": rfc3339(now - time::Duration::minutes(1))?, + "spaceId": space, + "peerId": peer.into_string().await.context("peer DID")?, + }))?)?; + let host_signature = sign(&wallet, &host)?; + let host: SignedMessage = serde_json::from_value(json!({ + "siwe": host.to_string(), "signature": format!("0x{}", hex::encode(host_signature)), + }))?; + delegate(client, &siwe_to_delegation_headers(host)).await?; + + let key = ed25519_dalek::SigningKey::from_bytes(&[114; 32]); + let jwk: JWK = serde_json::from_value(json!({ + "kty": "OKP", "crv": "Ed25519", + "d": base64::encode_config(key.to_bytes(), base64::URL_SAFE_NO_PAD), + "x": base64::encode_config(key.verifying_key().to_bytes(), base64::URL_SAFE_NO_PAD), + }))?; + let prepared = prepare_session(serde_json::from_value(json!({ + "abilities": {"kv": {"": ["tinycloud.kv/put", "tinycloud.kv/get"]}}, + "address": format!("0x{}", encode_eip55(&address)), + "chainId": 1, "domain": "localhost", + "issuedAt": rfc3339(now - time::Duration::minutes(1))?, + "expirationTime": rfc3339(now + time::Duration::hours(1))?, + "spaceId": space, "jwk": jwk, "nonce": "tc887s3bnode", + }))?)?; + let signature = sign(&wallet, &prepared.siwe)?; + let session = complete_session_setup(SignedSession { + session: prepared, + signature: Signature::from(signature), + })?; + delegate(client, &session.delegation_header).await?; + Ok(session) +} + +async fn invoke( + client: &Client, + session: &Session, + path: &str, + action: &str, + body: &str, +) -> Result<(Status, String)> { + let resource = session.space_id.clone().to_resource( + "kv".parse::()?, + Some(path.parse::()?), + None, + None, + ); + let invocation = session.invoke_any([(resource, [action.parse::()?])], None)?; + let response = client + .post("/invoke") + .header(Header::new("Authorization", invocation.encode()?)) + .header(ContentType::Plain) + .body(body) + .dispatch() + .await; + Ok(( + response.status(), + response.into_string().await.unwrap_or_default(), + )) +} + +fn files(root: &FsPath) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push(path); + } + } + } + out +} + +fn contains(bytes: &[u8], needle: &[u8]) -> bool { + bytes.windows(needle.len()).any(|w| w == needle) +} + +/// A short value and a multi-segment one (200 000 bytes, > 3 × 64 KiB). +fn values() -> Vec<(String, String)> { + let big: String = std::iter::once(MARKER.to_string()) + .chain((0..20_000).map(|i| format!("{:09} ", i))) + .collect(); + vec![ + ("notes/short".into(), format!("{MARKER} short value")), + ("notes/big".into(), big), + ] +} + +/// Also the regression test for a SQLite deadlock: the first put in a new +/// space creates the space's data key, which must happen before the +/// invocation's write lock and transaction (`prepare_write`), not inside them. +#[tokio::test] +async fn an_encrypting_filesystem_node_seals_kv_values_and_serves_plaintext() -> Result<()> { + tokio::time::timeout(std::time::Duration::from_secs(120), encrypting_node()) + .await + .context("encrypting node test timed out (first-put deadlock?)")? +} + +async fn encrypting_node() -> Result<()> { + let dir = tempfile::tempdir()?; + let at_rest = json!({"mode": "encrypt", "escrow_none": true}); + { + let client = node(dir.path(), at_rest.clone()).await?; + let session = session(&client).await?; + for (path, value) in values() { + let (status, body) = + invoke(&client, &session, &path, "tinycloud.kv/put", &value).await?; + ensure!(status == Status::Ok, "put {path}: {status} {body}"); + let (status, body) = invoke(&client, &session, &path, "tinycloud.kv/get", "").await?; + ensure!( + status == Status::Ok && body == value, + "get {path}: {status} {} bytes", + body.len() + ); + } + } + + let blocks = dir.path().join("blocks"); + let stored = files(&blocks); + ensure!( + stored.len() == 2, + "expected 2 stored blocks, got {stored:?}" + ); + let (_, address, space) = wallet()?; + let address = hex::encode(address); + for file in &stored { + let rel = file.strip_prefix(&blocks)?.to_string_lossy().to_lowercase(); + ensure!(rel.starts_with("v1/"), "not a sealed name: {rel}"); + ensure!( + !rel.contains(&address) && !rel.contains("at-rest-fs"), + "name leaks the space: {rel}" + ); + let raw = std::fs::read(file)?; + ensure!( + !contains(&raw, MARKER.as_bytes()), + "plaintext marker on disk in {rel}" + ); + ensure!(raw.starts_with(b"TCE1"), "{rel} is not TCE1"); + } + println!("sealed blocks for {space}: {stored:?}"); + + // Restart with the same root: the keys come back from the database and + // the sealed values read back. + let client = node(dir.path(), at_rest).await?; + let session = session(&client).await?; + for (path, value) in values() { + let (status, body) = invoke(&client, &session, &path, "tinycloud.kv/get", "").await?; + ensure!( + status == Status::Ok && body == value, + "after restart, get {path}: {status}" + ); + } + Ok(()) +} + +#[tokio::test] +async fn an_off_node_still_stores_plaintext_under_legacy_names() -> Result<()> { + let dir = tempfile::tempdir()?; + let client = node(dir.path(), json!({})).await?; + let session = session(&client).await?; + let (path, value) = values().remove(0); + let (status, body) = invoke(&client, &session, &path, "tinycloud.kv/put", &value).await?; + ensure!(status == Status::Ok, "put: {status} {body}"); + let stored = files(&dir.path().join("blocks")); + ensure!(stored.len() == 1, "{stored:?}"); + ensure!(!stored[0].to_string_lossy().contains("/v1/"), "{stored:?}"); + ensure!( + contains(&std::fs::read(&stored[0])?, MARKER.as_bytes()), + "off mode keeps plaintext" + ); + Ok(()) +} diff --git a/tinycloud-node-server/tests/did_resolution_single_entry.rs b/tinycloud-node-server/tests/did_resolution_single_entry.rs index dd6f3719..3dbbf8f0 100644 --- a/tinycloud-node-server/tests/did_resolution_single_entry.rs +++ b/tinycloud-node-server/tests/did_resolution_single_entry.rs @@ -144,7 +144,7 @@ impl Guard { | ("tinycloud-core/src/manifest.rs", "id_from_vm", "r . resolve (did)") | ("tinycloud-core/src/share_email/ports.rs", "resolve_exact", "self . resolve (policy , delegation)") | ("tinycloud-core/src/storage/memory.rs", "read_range", "range . resolve (total_size)") - | ("tinycloud-node-server/src/storage/file_system.rs", "read_range", "range . resolve (total_size)") + | ("tinycloud-node-server/src/storage/file_system.rs", "range_legacy", "range . resolve (total_size)") | ("tinycloud-node-server/src/storage/s3.rs", "parse_content_range", "ByteRangeSpec :: Inclusive { start , end } . resolve (total_size)") // TC-887: at-rest Range mapping resolves an HTTP byte range against the plaintext length. | ("tinycloud-core/src/at_rest/stream.rs", "map_range", "spec . resolve (self . plaintext_len ())") From e39fbef38bc160f89261d39ba8b5b5aab2676c8b Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 10:20:48 +0000 Subject: [PATCH 2/5] fix(TC-887): keep the FileSystem store's read and write futures small Box the decrypting reader inside FsReadable and the sealed read, range and persist futures behind the async-trait methods. The larger inline futures overflowed the 2 MiB test-thread stack in policy_v3::tests::policy_owner_may_be_a_session_the_space_owner_delegated_to. --- .../src/storage/file_system.rs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index 842e312d..e2fb124c 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -186,7 +186,10 @@ impl FileSystemStore { let (size, reader) = open_sealed(provider, space, id, file.take(len).compat(), len) .await? .into_inner(); - Ok(Some(Content::new(size, AsyncEither::Right(reader)))) + Ok(Some(Content::new( + size, + AsyncEither::Right(Box::new(reader)), + ))) } async fn read_block( @@ -285,7 +288,7 @@ impl FileSystemStore { Ok(Some(RangeRead::Content { total_size: plan.plaintext_len(), range, - content: Content::new(range.len(), AsyncEither::Right(reader)), + content: Content::new(range.len(), AsyncEither::Right(Box::new(reader))), })) } @@ -338,7 +341,9 @@ impl FileSystemStore { } /// Plaintext from a legacy file, or authenticated plaintext from a sealed one. -pub type FsReadable = AsyncEither>; +/// The decrypting side is boxed: it carries the AES key schedule and segment +/// buffers, and `Content` values move by value through deep request paths. +pub type FsReadable = AsyncEither>>; #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct FileSystemConfig { @@ -420,7 +425,9 @@ impl ImmutableReadStore for FileSystemStore { id: &Hash, ) -> Result>, Self::Error> { let start = Instant::now(); - let result = self.read_block(space, id).await; + // Boxed: the sealed read path's future is large, and the trait method + // future is built on the caller's stack before async_trait boxes it. + let result = Box::pin(self.read_block(space, id)).await; crate::prometheus::observe_stage( crate::prometheus::InvocationStage::BlockRead, crate::prometheus::StageOutcome::from(result.is_ok()), @@ -435,7 +442,7 @@ impl ImmutableReadStore for FileSystemStore { id: &Hash, range: ByteRangeSpec, ) -> Result>, Self::Error> { - self.range_block(space, id, range).await + Box::pin(self.range_block(space, id, range)).await } } @@ -599,7 +606,7 @@ impl ImmutableWriteStore for FileSystemStore { let result = async move { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - self.persist_staged(space, &hash, Staged::File(f)).await?; + Box::pin(self.persist_staged(space, &hash, Staged::File(f))).await?; Ok(hash) } .await; @@ -627,7 +634,7 @@ impl ImmutableWriteStore for FileSystemStore { let result = async move { let (mut h, v) = staged.into_inner(); let hash = h.finalize(); - self.persist_staged(space, &hash, Staged::Bytes(v)).await?; + Box::pin(self.persist_staged(space, &hash, Staged::Bytes(v))).await?; Ok(hash) } .await; @@ -661,7 +668,7 @@ impl ImmutableWriteStore Staged::File(t_file), AsyncEither::Right(v) => Staged::Bytes(v), }; - self.persist_staged(space, &hash, staged).await?; + Box::pin(self.persist_staged(space, &hash, staged)).await?; Ok(hash) } .await; From e71172ab2d9530c667959d47dae164fecaece085 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 12:28:57 +0000 Subject: [PATCH 3/5] feat(TC-887): the FileSystem store declares its seal path On S1 round 3, at_rest::setup refuses a non-off mode until storage declares a seal path. storage::attach_at_rest declares it for the FileSystem store, which seals every write; S3 declares none yet, so its own refusal goes and setup's applies. The ceremony test now expects encrypt and strict to boot and encrypt with a pinned, verified escrow; the docs no longer say no storage seals. Drops the duplicate parking_lot dev-dependency. --- docs/at-rest-escrow.md | 2 +- tinycloud-node-server/Cargo.toml | 2 -- tinycloud-node-server/src/at_rest.rs | 27 ++++++++++--------- tinycloud-node-server/src/lib.rs | 2 +- tinycloud-node-server/src/storage/mod.rs | 19 ++++++------- .../tests/at_rest_ceremony.rs | 23 ++++++---------- 6 files changed, 33 insertions(+), 42 deletions(-) diff --git a/docs/at-rest-escrow.md b/docs/at-rest-escrow.md index 74b7cf4b..a083ef3c 100644 --- a/docs/at-rest-escrow.md +++ b/docs/at-rest-escrow.md @@ -146,7 +146,7 @@ The same settings as compose env: `TINYCLOUD_AT_REST__MODE`, `TINYCLOUD_AT_REST_ - it keeps a copy of the sealed key, the bundle, the ceremony record and each space's wrapped data key in the block store (`_tc/at-rest/` in S3, `{datadir}/at-rest/_tc/at-rest/` on disk). Data-key objects are named by a keyed hash, not the space; - in this mode a copy that can't be written is logged and the boot goes on (nothing depends on it yet). With `encrypt` or `strict`, every boot-time copy must be written: a block store that refuses the write stops the boot; - nothing is encrypted. -- **`encrypt` and `strict` are refused at boot in this build.** Block storage has no seal path yet, so the node won't claim to encrypt while it writes plaintext; `/info` never shows `encrypting: true`. Storage turns the capability on when the seal path lands (TC-887 S3). From then on, `encrypt` or `strict` refuses to start unless the recipients equal the bundle's, `verified_fp` equals its EFP, and the stored ceremony record verifies under the key inside the TEE. A Static-root node with no recipients may use `escrow_none = true` instead, which acknowledges that no escrow exists. +- **`encrypt` and `strict` need a block store with a seal path.** The FileSystem store seals KV blocks (TC-887 S3); with S3 block storage, `encrypt` and `strict` are refused at boot in this build, so the node won't claim to encrypt while it writes plaintext. With a seal path, `encrypt` or `strict` refuses to start unless the recipients equal the bundle's, `verified_fp` equals its EFP, and the stored ceremony record verifies under the key inside the TEE. A Static-root node with no recipients may use `escrow_none = true` instead, which acknowledges that no escrow exists. - **In a TEE, the escrow recipients and `verified_fp` must be literal values in the measured compose.** With any at-rest config, the node reads its `app-compose.json` from the guest agent (`/Info`, `tcb_info.app_compose`), checks it against `compose_hash`, and refuses to boot unless `TINYCLOUD_AT_REST__ESCROW__RECIPIENTS` and `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP` are set there as literals (no `${…}`, no pass-through) equal to the effective config. Values from the encrypted environment, a file or anywhere else are refused. If the app-compose can't be read or parsed, the node refuses to boot. The `/Info` shape must be confirmed on a live CVM before the escrow ceremony. - **The escrow bundle is made or changed only at boot, never by a request.** Every endpoint below only reads it. - **An ARK never gets a second bundle for other holders.** The node refuses to boot (`escrow_holders_changed`) if the configured recipients differ from any existing bundle for its ARK: the active bundle, any stored escrow row, a ceremony record in the block store whose MAC verifies under the ARK, or a bundle object in the block store naming the ARK. This holds after a restore too, when the database lost its escrow rows. The block-store bundle object is unauthenticated, so it can only refuse, never be used. Changing a holder after the ceremony needs an ARK rotation (TC-1345). diff --git a/tinycloud-node-server/Cargo.toml b/tinycloud-node-server/Cargo.toml index 435a6ed8..0b1a61c0 100644 --- a/tinycloud-node-server/Cargo.toml +++ b/tinycloud-node-server/Cargo.toml @@ -138,8 +138,6 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd # zlib compressibility of raw stored bytes in tests/at_rest_harness.rs (TC-887). # Already in the lock through ssi-status; adds no new crate. flate2 = "1" -# TC-887: a switchable at-rest state in the sealed block-store tests. -parking_lot = "0.12" k256 = { version = "0.13.4", features = ["ecdsa"] } sha3 = "0.10" # The Rust half of the JS SDK: builds the same host and session SIWE diff --git a/tinycloud-node-server/src/at_rest.rs b/tinycloud-node-server/src/at_rest.rs index 238a9fca..4d696248 100644 --- a/tinycloud-node-server/src/at_rest.rs +++ b/tinycloud-node-server/src/at_rest.rs @@ -5,9 +5,11 @@ //! keyring needs its tables: [`AtRestRuntime::new`] before storage, then //! [`setup`] after migrations installs the keyring. //! -//! In this build no storage seals anything yet: nothing calls -//! [`AtRestRuntime::declare_seal_path`], so `setup` refuses every mode -//! but `off` and `encrypting()` is always false. +//! Block storage declares its seal path when it is attached +//! ([`crate::storage::attach_at_rest`] calls +//! [`AtRestRuntime::declare_seal_path`] for a store that seals every +//! write); without it `setup` refuses every mode but `off` and +//! `encrypting()` stays false. //! //! With the default config (`mode = "off"`, no recipients, no //! `escrow_none`) nothing is set up: no key reads, no rows, no mirror. @@ -32,14 +34,14 @@ use crate::config::{AtRestConfig, AtRestEscrowConfig, AtRestMode, Storage}; use crate::storage::at_rest_mirror::{FsKeyMirror, S3KeyMirror}; use crate::tee::TeeContext; -/// At-rest state shared by the routes and `/info` (and, once it seals -/// writes, block storage). Rocket manages it as `Arc`. +/// At-rest state shared by the routes, `/info` and block storage. Rocket +/// manages it as `Arc`. /// -/// The contract for the storage seal path (TC-887 S3), not yet wired: mode -/// `off` behaves exactly as without at-rest. Otherwise writes are sealed -/// only while [`encrypting`](Self::encrypting) (else 503, never plaintext); -/// a sealed read whose key is `Locked` is a 503; legacy plaintext reads are -/// allowed unless the mode is `strict`. +/// The contract for the storage seal path (TC-887 S3): mode `off` behaves +/// exactly as without at-rest. Otherwise writes are sealed only while +/// [`encrypting`](Self::encrypting) (else 503, never plaintext); a sealed +/// read whose key is `Locked` is a 503; legacy plaintext reads are allowed +/// unless the mode is `strict`. pub struct AtRestRuntime { mode: AtRestMode, escrow_none: bool, @@ -48,8 +50,9 @@ pub struct AtRestRuntime { /// writes may be encrypted. Set at boot, or after a recovery unlocks a /// node whose gate then passes. encrypting: AtomicBool, - /// Set by block storage when it seals writes (TC-887 S3). Until then a - /// mode other than `off` is refused at boot and `encrypting` stays false. + /// Set by block storage when it is attached with a seal path (TC-887 + /// S3). Without it a mode other than `off` is refused at boot and + /// `encrypting` stays false. seal_path: AtomicBool, share_failures: ShareFailureLimiter, } diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 55a0aadd..08a8479c 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -650,7 +650,7 @@ pub async fn app_with_control( // it; the keyring is installed after migrations (`at_rest::setup`). let at_rest_runtime = at_rest::AtRestRuntime::new(&tinycloud_config.at_rest); let mut blocks = tinycloud_config.storage.blocks.open().await?; - storage::attach_at_rest(&mut blocks, at_rest_runtime.clone())?; + storage::attach_at_rest(&mut blocks, at_rest_runtime.clone()); let tinycloud = TinyCloud::new_with_encryption( database_connection, diff --git a/tinycloud-node-server/src/storage/mod.rs b/tinycloud-node-server/src/storage/mod.rs index a10fa30c..853e4d62 100644 --- a/tinycloud-node-server/src/storage/mod.rs +++ b/tinycloud-node-server/src/storage/mod.rs @@ -6,24 +6,21 @@ pub mod size; use std::sync::Arc; -use crate::{at_rest::AtRestRuntime, config::AtRestMode, BlockStores}; +use crate::{at_rest::AtRestRuntime, BlockStores}; use tinycloud_core::storage::either::Either; /// Links the opened block store to the node's at-rest state (TC-887). /// -/// The FileSystem store seals KV blocks when the mode and keys allow. The S3 -/// store does not seal yet, so a non-`off` mode with S3 storage refuses to -/// boot rather than write plaintext. -pub fn attach_at_rest(stores: &mut BlockStores, runtime: Arc) -> anyhow::Result<()> { +/// The FileSystem store seals KV blocks when the mode and keys allow, so it +/// declares a seal path. The S3 store does not seal yet and declares none, +/// so `at_rest::setup` refuses a non-`off` mode with S3 storage rather than +/// write plaintext. +pub fn attach_at_rest(stores: &mut BlockStores, runtime: Arc) { match stores { Either::B(fs) => { + runtime.declare_seal_path(); fs.attach_at_rest(sealed::AtRestHandle::new(runtime)); - Ok(()) } - Either::A(_) if runtime.mode() == AtRestMode::Off => Ok(()), - Either::A(_) => anyhow::bail!( - "at_rest.mode = \"{}\" is not supported with S3 block storage in this build", - runtime.mode_name() - ), + Either::A(_) => {} } } diff --git a/tinycloud-node-server/tests/at_rest_ceremony.rs b/tinycloud-node-server/tests/at_rest_ceremony.rs index 0e9de195..1574ab3f 100644 --- a/tinycloud-node-server/tests/at_rest_ceremony.rs +++ b/tinycloud-node-server/tests/at_rest_ceremony.rs @@ -191,17 +191,19 @@ async fn ceremony_gate_and_recovery_through_the_app() { ); drop(client); - // No seal path in this build: encrypt and strict are refused at boot, - // even with the escrow pinned and verified, so nothing ever claims to - // encrypt while storage writes plaintext. + // The FileSystem store seals every write: with the escrow pinned and + // verified, encrypt and strict boot and encrypt. for mode in ["encrypt", "strict"] { - let err = refused( + let client = app( dir.path(), 1, json!({"mode": mode, "escrow": escrow(Some(&efp))}), ) - .await; - assert!(err.contains("no at-rest seal path"), "{err}"); + .await + .unwrap(); + let (_, st) = get_json(&client, "/at-rest/status").await; + assert_eq!(st["encrypting"], json!(true), "{mode}"); + drop(client); } // Pinned and verified, mode off: the gate passes, nothing is encrypted. @@ -358,15 +360,6 @@ async fn misconfigurations_are_refused_at_boot() { ) .await; assert!(err.contains("not both"), "{err}"); - // No seal path: encrypt is refused even on a self-hosted Static root - // that acknowledges having no escrow. - let err = refused( - dir.path(), - 1, - json!({"mode": "encrypt", "escrow_none": true}), - ) - .await; - assert!(err.contains("no at-rest seal path"), "{err}"); // Unknown keys in the section are refused, not ignored. refused(dir.path(), 1, json!({"escrow_nonee": true})).await; } From 0cc5ce4c3ae9b73f0e20d3a2f6404115b15a88dd Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 14:09:41 +0000 Subject: [PATCH 4/5] fix(TC-887): authorize and prepare every touched space before the write lock; keys only looked up under it (S3 review) --- docs/at-rest/INTERFACES.md | 42 ++++- tinycloud-core/src/db.rs | 139 ++++++++++++--- tinycloud-core/src/storage/either.rs | 12 ++ tinycloud-core/src/storage/mod.rs | 29 +++- tinycloud-node-server/src/routes/mod.rs | 12 +- .../src/storage/file_system.rs | 41 +++-- .../src/storage/file_system/sealed_tests.rs | 156 +++++++++++++++++ tinycloud-node-server/src/storage/sealed.rs | 164 ++++++++++++++++-- .../src/storage/sealed/tests.rs | 114 +++++++++--- .../tests/at_rest_fs_node.rs | 126 +++++++++++++- 10 files changed, 741 insertions(+), 94 deletions(-) diff --git a/docs/at-rest/INTERFACES.md b/docs/at-rest/INTERFACES.md index 198de792..d7a4e9ba 100644 --- a/docs/at-rest/INTERFACES.md +++ b/docs/at-rest/INTERFACES.md @@ -65,10 +65,22 @@ pub trait DataKeyProvider: Send + Sync { /// by deliberate crypto-shred). async fn key(&self, space: &str, dek_id: DekId) -> Result; + /// Before work that may read `space`'s sealed blobs under the invoke write + /// lock: put back any of the space's keys the database lost (from an + /// authenticated mirror). Never creates a key. Locked keys are not an error + /// here (the read itself answers 503). + async fn prepare_read(&self, space: &str) -> Result<(), DataKeyError>; + /// NK for sealed S3 object names. Same value for the life of the deployment. fn object_name_key(&self) -> Result; } +/// Work under the invoke write lock and transaction runs inside +/// `write_locked(..)`; there `current_key`/`key` make no key-store writes and +/// answer `NotPrepared` for a key that would need creating or restoring. +pub async fn write_locked(f: F) -> F::Output; +pub fn in_write_lock() -> bool; + #[derive(Debug, thiserror::Error)] pub enum DataKeyError { /// Node booted locked (W cannot unseal ARK). Callers map to HTTP 503. @@ -81,6 +93,9 @@ pub enum DataKeyError { /// PG/S3/unwrap failure (incl. a wrapped DEK failing AEAD). Opaque, no key bytes. #[error("data key backend: {0}")] Backend(#[source] Box), + /// A key needed creating or restoring inside `write_locked`. HTTP 503. + #[error("the space's data key was not prepared before the write lock")] + NotPrepared, } ``` @@ -290,8 +305,29 @@ redacted. counts legacy + sealed. While locked, a space reports legacy bytes only (writes are 503 then). A sealed object with an invalid TCE1 length counts 0 bytes; the warning is rate-limited and carries only the length. -- **Errors:** at-rest causes map explicitly to 503 (keys unavailable / key store) or 500 - (integrity, unsupported, legacy refused in strict) on `/invoke` and the KV GET routes. The - general "store error → 401" catch-all is tracked separately (TC-1347). +- **Errors:** at-rest causes map explicitly to 503 (keys unavailable, key store unavailable, key + not prepared) or 500 (integrity, unsupported, legacy refused in strict, inconsistent key store) + on `/invoke` and the KV GET routes. Bodies carry reason codes only; backend detail goes to the + server log. Integrity and unsupported failures are logged (typed, rate-limited, no names), + including ones that abort a response mid-stream. The general "store error → 401" catch-all is + tracked separately (TC-1347). +- **Prepare before the write lock:** while keys are set up, a writing invocation is authorized and + its spaces' hosting checked *before* any key work, so a refused request creates no data key or + mirror object. Then every space it touches is prepared: a data key the database lost is restored + from its authenticated mirror (never created), and written spaces get their current key. Inside + the invoke write lock and transaction the provider only looks keys up (`at_rest::write_locked`); + a key that would need creating or restoring there is `NotPrepared` (503), never a key-store + write, which on SQLite would wait on the lock the request holds. +- **Accepted residual (linkability during the transition):** in `encrypt`, a read of a + legacy-only block asks for the sealed name first, then the legacy name (`strict` probes the legacy + name to answer "legacy refused"). A storage provider's access logs can therefore link a + `space_tag` to its space id (and an object tag to its content hash) for every space read before + the backfill, for the life of the object-name key. +- **Downgrade window until `strict`:** legacy objects are served without a hash check. In `off` or + `encrypt`, someone who can write the bucket and delete sealed objects can serve any bytes under the + legacy names they know. `strict` (after the backfill) closes this. +- **`remove`:** deletes the sealed copy when keys can name it, else the legacy copy. When both + exist, the legacy copy survives, and while keys are locked only the legacy copy is removed. KV + delete is logical, so nothing calls it in production today; S5/GC must delete both names. - **Reserved names:** The key-continuity key mirror lives under `_tc/` in the block bucket (`datadir/at-rest` on FS). The block boot scan skips it. diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index 415077cb..4d6be576 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -2173,17 +2173,68 @@ where // every earlier same-space commit and no same-space writer can commit // until this one does. A failed precondition is a 412, never a false // success (see `postgres_conditional_put_is_atomic_under_concurrency`). - // TC-887: let the block store prepare each written space (e.g. create - // its at-rest data key) before the write lock and transaction below; - // `persist` runs inside them, where such a write would deadlock on - // SQLite. A failure here fails the request before the transaction. - let written_spaces: std::collections::HashSet<&SpaceId> = - stages.keys().map(|(space, _)| space).collect(); - for space in written_spaces { - self.storage - .prepare_write(space) - .await - .map_err(TxStoreError::StoreWrite)?; + // TC-887: a block store with per-space state for reads and writes + // (at-rest data keys) prepares every space the invocation touches + // here, before the write lock and transaction: inside them its own + // database writes would wait, on SQLite, on the lock this request + // holds, so the reads and `persist` below only look keys up. Only an + // authorized invocation on hosted spaces gets this far, so a refused + // request prepares (creates) nothing; the transaction still + // authorizes it as before. A failure fails the request here. Boxed: + // the authorization futures would otherwise grow every invocation's + // future, which runs on the caller's stack. + if self.storage.prepares() { + Box::pin(async { + match mode { + InvokeMode::Admitted => invocation::authorize_admitted( + &self.conn, + &invocation.0, + OffsetDateTime::now_utc(), + None, + ) + .await + .map_err(TxError::::from)?, + InvokeMode::Public => invocation::verify_and_authorize_with_graph( + &self.conn, + &invocation.0, + OffsetDateTime::now_utc(), + None, + ) + .await + .map_err(TxError::::from)?, + // Node-made invocations carry no delegation chain; the + // transaction accepts them as internal events. + InvokeMode::Internal | InvokeMode::MeetingPublication => {} + } + let touched = invocation.0.spaces().collect::>(); + let hosted = space::Entity::find() + .filter( + space::Column::Id.is_in(touched.iter().map(|s| SpaceIdWrap((*s).clone()))), + ) + .count(&self.conn) + .await?; + if hosted != touched.len() as u64 { + return Err(TxError::SpaceNotFound.into()); + } + // Every touched space may be read inside the transaction (a + // get, or a written space's existing blocks); written ones are + // also made ready for `persist`. + let written: HashSet<&SpaceId> = stages.keys().map(|(space, _)| space).collect(); + for space in touched { + self.storage + .prepare_read(space) + .await + .map_err(TxStoreError::StoreRead)?; + if written.contains(space) { + self.storage + .prepare_write(space) + .await + .map_err(TxStoreError::StoreWrite)?; + } + } + Ok::<(), TxStoreError>(()) + }) + .await?; } let sequence_spaces = invocation.0.spaces().cloned().collect::>(); let _sequence_guards = self.acquire_space_sequence_guards(&sequence_spaces).await; @@ -2313,8 +2364,10 @@ where }) { match cap { (space, "kv", "tinycloud.kv/get", path) => { + // Under the write lock: the store only looks keys up + // here (the space was prepared above). let data = - get_kv(&tx, &self.storage, space, path) + crate::at_rest::write_locked(get_kv(&tx, &self.storage, space, path)) .await .map_err(|e| match e { EitherError::A(e) => TxStoreError::Tx(e.into()), @@ -2352,8 +2405,7 @@ where } (space, "kv", "tinycloud.kv/put", path) => { if let Some(stage) = stages.remove(&(space.clone(), path.clone())) { - self.storage - .persist(space, stage) + crate::at_rest::write_locked(self.storage.persist(space, stage)) .await .map_err(TxStoreError::StoreWrite)?; let hash = write_hashes @@ -5088,19 +5140,22 @@ mod test { self.inner.persist(space, stage).await } } - /// Records whether the SQLite writer lock was free when each write-path - /// hook ran, and can fail `persist`. + /// Records, for each storage hook, whether the SQLite writer lock was free + /// and whether it ran inside `at_rest::write_locked`; can fail `persist`. + /// (hook, writer lock free, inside `write_locked`). + type HookLog = Arc>>; #[derive(Clone, Default)] struct LockProbeStore { inner: MemoryStore, lock: Arc>>>, - log: Arc>>, + log: HookLog, fail_persist: bool, } impl LockProbeStore { fn record(&self, hook: &'static str) { let free = self.lock.get().expect("lock installed").try_lock().is_ok(); - self.log.lock().push((hook, free)); + let scoped = crate::at_rest::in_write_lock(); + self.log.lock().push((hook, free, scoped)); } } #[async_trait::async_trait] @@ -5117,6 +5172,13 @@ mod test { async fn contains(&self, space: &SpaceId, hash: &Hash) -> Result { self.inner.contains(space, hash).await } + fn prepares(&self) -> bool { + true + } + async fn prepare_read(&self, _space: &SpaceId) -> Result<(), Self::Error> { + self.record("prepare_read"); + Ok(()) + } async fn read( &self, space: &SpaceId, @@ -5153,12 +5215,15 @@ mod test { } } - /// TC-887 invariant: a block store's own database writes (e.g. creating a - /// space's at-rest data key) happen in `prepare_write`, which runs before - /// the invocation takes the SQLite writer lock; `persist` runs while it is - /// held. A key-store write in `persist` would deadlock a SQLite node. + /// TC-887 invariant: a block store's own database writes (e.g. creating or + /// restoring a space's at-rest data key) happen in `prepare_read` and + /// `prepare_write`, which run after authorization and before the + /// invocation takes the SQLite writer lock; `persist` runs while it is + /// held, inside `at_rest::write_locked` (key lookups only). A key-store + /// write under the lock would deadlock a SQLite node. A space the node + /// does not host is refused before any hook runs. #[tokio::test] - async fn prepare_write_runs_before_the_writer_lock_and_persist_inside_it() { + async fn storage_prepares_hosted_spaces_before_the_writer_lock_and_persists_write_locked() { use crate::storage::memory::MemoryStaging; use futures::io::AsyncWriteExt; for fail_persist in [false, true] { @@ -5182,6 +5247,26 @@ mod test { .sqlite_writer_lock() .expect("a SQLite file node has a writer lock"); assert!(storage.lock.set(lock).is_ok()); + let unhosted = test_space_id("prepare-write-probe-unhosted"); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"probe").await.unwrap(); + let refused = db + .invoke_internal_kv_put::( + unhosted, + "probe/value".parse().unwrap(), + Metadata(Default::default()), + stage, + None, + ) + .await; + assert!( + matches!(refused, Err(TxStoreError::Tx(TxError::SpaceNotFound))), + "an unhosted space is refused" + ); + assert!( + storage.log.lock().is_empty(), + "a refused invocation prepares nothing" + ); let space = test_space_id("prepare-write-probe"); space::ActiveModel { id: Set(SpaceIdWrap(space.clone())), @@ -5206,8 +5291,12 @@ mod test { .expect("kv put must not hang"); assert_eq!( *storage.log.lock(), - vec![("prepare_write", true), ("persist", false)], - "prepare_write must see the lock free and persist must run under it" + vec![ + ("prepare_read", true, false), + ("prepare_write", true, false), + ("persist", false, true) + ], + "prepare hooks must see the lock free; persist must run under it, write-locked" ); if fail_persist { // The transaction rolls back after a successful prepare: no KV diff --git a/tinycloud-core/src/storage/either.rs b/tinycloud-core/src/storage/either.rs index 0c2020bd..7dbefa35 100644 --- a/tinycloud-core/src/storage/either.rs +++ b/tinycloud-core/src/storage/either.rs @@ -31,6 +31,18 @@ where Self::B(r) => r.contains(space, id).await.map_err(Self::Error::B), } } + fn prepares(&self) -> bool { + match self { + Self::A(l) => l.prepares(), + Self::B(r) => r.prepares(), + } + } + async fn prepare_read(&self, space: &SpaceId) -> Result<(), Self::Error> { + match self { + Self::A(l) => l.prepare_read(space).await.map_err(Self::Error::A), + Self::B(r) => r.prepare_read(space).await.map_err(Self::Error::B), + } + } async fn read( &self, space: &SpaceId, diff --git a/tinycloud-core/src/storage/mod.rs b/tinycloud-core/src/storage/mod.rs index 8ca0a1de..81169566 100644 --- a/tinycloud-core/src/storage/mod.rs +++ b/tinycloud-core/src/storage/mod.rs @@ -125,6 +125,22 @@ pub trait ImmutableReadStore: Send + Sync { type Error: StdError + Send + Sync; type Readable: futures::io::AsyncRead + Send + Sync; async fn contains(&self, space: &SpaceId, id: &Hash) -> Result; + /// Whether a writing invocation must be authorized, and the spaces it + /// touches prepared ([`prepare_read`](Self::prepare_read), + /// [`ImmutableWriteStore::prepare_write`]), before its database write + /// lock and transaction. Default: no. + fn prepares(&self) -> bool { + false + } + /// When [`prepares`](Self::prepares): called for each space a writing + /// invocation touches without writing it, once the invocation is + /// authorized and before the write lock. Reads inside the transaction + /// then need no database writes (on SQLite they would wait on the lock + /// the invocation holds), e.g. restoring the space's at-rest data key. + /// Default: nothing. + async fn prepare_read(&self, _space: &SpaceId) -> Result<(), Self::Error> { + Ok(()) + } async fn read( &self, space: &SpaceId, @@ -175,9 +191,10 @@ where S::Writable: 'static, { type Error: StdError + Send + Sync; - /// Called for each space an invocation writes, before its database write - /// lock and transaction are taken; `persist` later runs inside them. A - /// store that needs its own database writes for a space (e.g. creating + /// When [`ImmutableReadStore::prepares`]: called for each space an + /// invocation writes, once the invocation is authorized and before its + /// database write lock and transaction; `persist` later runs inside them. + /// A store that needs its own database writes for a space (e.g. creating /// the space's at-rest data key) does them here. Default: nothing. async fn prepare_write(&self, _space: &SpaceId) -> Result<(), Self::Error> { Ok(()) @@ -223,6 +240,12 @@ where async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { self.contains(space, id).await } + fn prepares(&self) -> bool { + S::prepares(self) + } + async fn prepare_read(&self, space: &SpaceId) -> Result<(), Self::Error> { + S::prepare_read(self, space).await + } async fn read( &self, space: &SpaceId, diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index 2a27634f..71157c11 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -139,12 +139,15 @@ fn database_failure( /// TC-887: the status for an invocation whose block-store step failed for /// at-rest reasons (503 keys unavailable, 500 integrity/unsupported), if so. -fn at_rest_invoke_status( +pub(crate) fn at_rest_invoke_status( e: &TxStoreError, ) -> Option { match e { TxStoreError::StoreRead(err) | TxStoreError::StoreWrite(err) => { - crate::storage::sealed::block_store_at_rest_error(err).map(|e| e.status()) + crate::storage::sealed::block_store_at_rest_error(err).map(|e| { + e.log_failure(); + e.status() + }) } _ => None, } @@ -160,7 +163,10 @@ pub(crate) fn kv_read_failure( use tinycloud_core::storage::either::EitherError; let status = match &e { EitherError::B(err) => crate::storage::sealed::block_store_at_rest_error(err) - .map(|e| e.status()) + .map(|e| { + e.log_failure(); + e.status() + }) .unwrap_or(Status::InternalServerError), EitherError::A(_) => Status::InternalServerError, }; diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index e2fb124c..f72062d8 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -1,8 +1,8 @@ use super::{ sealed::{ - open_sealed, plan_range, read_plan, seal_file, seal_vec, space_tag_of, write_target, - AtRestHandle, AtRestStoreError, AtRestView, ReadPlan, SealFileError, SealedName, - WriteTarget, + contains_name, open_sealed, plan_range, read_plan, seal_file, seal_vec, space_tag_of, + write_target, AtRestHandle, AtRestStoreError, AtRestView, ContainsName, LoggedDecrypt, + ReadPlan, SealFileError, SealedName, WriteTarget, }, size::SpaceSizes, }; @@ -24,7 +24,7 @@ use std::{ use tempfile::{NamedTempFile, PathPersistError}; use tinycloud_auth::{resource::SpaceId, ssi::dids::DIDBuf}; use tinycloud_core::{ - at_rest::{plaintext_len, DataKeyProvider, DecryptReader, HEADER_LEN}, + at_rest::{plaintext_len, DataKeyProvider, HEADER_LEN}, hash::Hash, storage::*, }; @@ -140,8 +140,10 @@ impl FileSystemStore { Staged::Bytes(v) => { let size = v.len() as u64; let ct = seal_vec(key, space, hash, v).await?; - let mut out = NamedTempFile::new_in(dir)?; - std::io::Write::write_all(&mut out, &ct)?; + let out = NamedTempFile::new_in(dir)?; + let mut file = File::from_std(out.reopen()?); + tokio::io::AsyncWriteExt::write_all(&mut file, &ct).await?; + tokio::io::AsyncWriteExt::flush(&mut file).await?; (size, out) } }; @@ -188,7 +190,7 @@ impl FileSystemStore { .into_inner(); Ok(Some(Content::new( size, - AsyncEither::Right(Box::new(reader)), + AsyncEither::Right(Box::new(LoggedDecrypt::new(reader))), ))) } @@ -288,7 +290,10 @@ impl FileSystemStore { Ok(Some(RangeRead::Content { total_size: plan.plaintext_len(), range, - content: Content::new(range.len(), AsyncEither::Right(Box::new(reader))), + content: Content::new( + range.len(), + AsyncEither::Right(Box::new(LoggedDecrypt::new(reader))), + ), })) } @@ -328,14 +333,12 @@ impl FileSystemStore { } } - /// Whether the block exists under a name this node may read. + /// Whether the block exists under the name a write would use (INTERFACES + /// §7): in `encrypt`/`strict` only the sealed name counts. fn exists(&self, space: &SpaceId, id: &Hash) -> Result { - Ok(match read_plan(&self.view(), space, id)? { - ReadPlan::Legacy | ReadPlan::LegacyWhileLocked => self.get_path(space, id).exists(), - ReadPlan::SealedThenLegacy { name, .. } => { - self.sealed_path(&name).exists() || self.get_path(space, id).exists() - } - ReadPlan::Sealed { name, .. } => self.sealed_path(&name).exists(), + Ok(match contains_name(&self.view(), space, id)? { + ContainsName::Legacy => self.get_path(space, id).exists(), + ContainsName::Sealed(name) => self.sealed_path(&name).exists(), }) } } @@ -343,7 +346,7 @@ impl FileSystemStore { /// Plaintext from a legacy file, or authenticated plaintext from a sealed one. /// The decrypting side is boxed: it carries the AES key schedule and segment /// buffers, and `Content` values move by value through deep request paths. -pub type FsReadable = AsyncEither>>; +pub type FsReadable = AsyncEither>>; #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct FileSystemConfig { @@ -419,6 +422,12 @@ impl ImmutableReadStore for FileSystemStore { async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { self.exists(space, id) } + fn prepares(&self) -> bool { + super::sealed::prepares(&self.view()) + } + async fn prepare_read(&self, space: &SpaceId) -> Result<(), Self::Error> { + Ok(super::sealed::prepare_read(&self.view(), space).await?) + } async fn read( &self, space: &SpaceId, diff --git a/tinycloud-node-server/src/storage/file_system/sealed_tests.rs b/tinycloud-node-server/src/storage/file_system/sealed_tests.rs index a36c231e..8eb5dc4c 100644 --- a/tinycloud-node-server/src/storage/file_system/sealed_tests.rs +++ b/tinycloud-node-server/src/storage/file_system/sealed_tests.rs @@ -260,6 +260,162 @@ async fn a_legacy_copy_does_not_suppress_the_sealed_write() { assert_eq!(read_all(&store, &h).await.unwrap().unwrap(), data); } +/// `contains` checks the name a write would use (INTERFACES §7): in encrypt +/// mode a legacy-only block does not count; once sealed it does. +#[tokio::test] +async fn contains_checks_the_name_a_write_would_use() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), AtRestView::off()).await; + let data = value(300); + let h = put_bytes(&store, &data).await.unwrap(); + assert!(store.contains(&space(), &h).await.unwrap()); + switch.set(encrypt()); + assert!(!store.contains(&space(), &h).await.unwrap(), "legacy only"); + put_bytes(&store, &data).await.unwrap(); + assert!(store.contains(&space(), &h).await.unwrap(), "sealed"); + switch.set(view(AtRestMode::Encrypt, false, Some(keys().locked()))); + let e = store.contains(&space(), &h).await.unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); +} + +/// Switching back to `off` with keys still reads sealed blocks; `off` with +/// locked keys answers 503 for a sealed-only block (it may exist but cannot +/// be named). +#[tokio::test] +async fn off_after_encrypt_reads_sealed_and_locked_off_is_unavailable() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), encrypt()).await; + let data = value(SEG + 3); + let h = put_bytes(&store, &data).await.unwrap(); + switch.set(view(AtRestMode::Off, false, Some(keys()))); + assert_eq!(read_all(&store, &h).await.unwrap().unwrap(), data); + switch.set(view(AtRestMode::Off, false, Some(keys().locked()))); + let e = store.read(&space(), &h).await.map(|_| ()).unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); +} + +/// `strict` with locked keys serves nothing: sealed and legacy reads are 503. +#[tokio::test] +async fn strict_with_locked_keys_is_unavailable_for_legacy_and_sealed() { + let dir = tempfile::tempdir().unwrap(); + let (store, switch) = open_store(dir.path(), AtRestView::off()).await; + let hl = put_bytes(&store, &value(40)).await.unwrap(); + switch.set(encrypt()); + let hs = put_bytes(&store, &value(41)).await.unwrap(); + switch.set(view(AtRestMode::Strict, false, Some(keys().locked()))); + for h in [hl, hs] { + let e = store.read(&space(), &h).await.map(|_| ()).unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); + let e = store + .read_range(&space(), &h, ByteRangeSpec::Suffix { length: 2 }) + .await + .map(|_| ()) + .unwrap_err(); + assert_eq!(status(&e), Some(Status::ServiceUnavailable), "{e}"); + } +} + +/// `From` and `Suffix` ranges at every segment edge (k·64 KiB ± 1, up to an +/// exact multiple) return the plaintext slice and plaintext totals, for both +/// staging paths. +#[tokio::test] +async fn from_and_suffix_ranges_match_the_plaintext_at_segment_edges() { + for (n, via_file) in [ + (3 * SEG, false), + (3 * SEG, true), + (2 * SEG + 1, true), + (2 * SEG - 1, false), + ] { + let dir = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), encrypt()).await; + let data = value(n); + let h = if via_file { + put_file(&store, &data).await.unwrap() + } else { + put_bytes(&store, &data).await.unwrap() + }; + let edges = [ + 0, + 1, + SEG - 1, + SEG, + SEG + 1, + 2 * SEG - 1, + 2 * SEG, + 2 * SEG + 1, + n - 1, + ]; + for &at in edges.iter().filter(|&&at| at < n) { + for (spec, want) in [ + (ByteRangeSpec::From { start: at as u64 }, &data[at..]), + ( + ByteRangeSpec::Suffix { + length: (n - at) as u64, + }, + &data[at..], + ), + ] { + match store.read_range(&space(), &h, spec).await.unwrap().unwrap() { + RangeRead::Content { + total_size, + content, + .. + } => { + assert_eq!(total_size, n as u64); + let mut out = Vec::new(); + Box::pin(content).read_to_end(&mut out).await.unwrap(); + assert!(out == want, "n={n} {spec:?}"); + } + RangeRead::Unsatisfiable { .. } => panic!("n={n} {spec:?} unsatisfiable"), + } + } + } + } +} + +/// A sealed persist that fails at its final step (the space's sealed +/// directory is read-only) leaves no temporary file behind: not the sealed +/// temp, not the staged plaintext (file staging, in its own directory), and +/// no legacy copy. +#[tokio::test] +async fn a_failed_sealed_persist_leaves_no_temporary_files() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let staging = tempfile::tempdir().unwrap(); + let (store, _) = open_store(dir.path(), encrypt()).await; + put_bytes(&store, b"creates the space's sealed directory") + .await + .unwrap(); + let first = sealed_files(dir.path()); + let tag_dir = first[0].parent().unwrap().to_path_buf(); + std::fs::set_permissions(&tag_dir, std::fs::Permissions::from_mode(0o555)).unwrap(); + + let data = value(SEG + 11); + let mut staged = HashBuffer::new(TempFileStage::new( + NamedTempFile::new_in(staging.path()).unwrap(), + )); + staged.write_all(&data).await.unwrap(); + staged.flush().await.unwrap(); + let via_file = + ImmutableWriteStore::::persist(&store, &space(), staged).await; + let via_bytes = put_bytes(&store, &data).await; + std::fs::set_permissions(&tag_dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + + assert!(via_file.is_err() && via_bytes.is_err()); + let staged_left: Vec<_> = std::fs::read_dir(staging.path()) + .unwrap() + .flatten() + .collect(); + assert!(staged_left.is_empty(), "{staged_left:?}"); + assert_eq!( + sealed_files(dir.path()), + first, + "no temporary or partial files" + ); + let h = tinycloud_core::hash::hash(&data); + assert!(!store.get_path(&space(), &h).exists()); +} + #[tokio::test] async fn locked_keys_refuse_writes_and_sealed_reads_but_serve_legacy() { let dir = tempfile::tempdir().unwrap(); diff --git a/tinycloud-node-server/src/storage/sealed.rs b/tinycloud-node-server/src/storage/sealed.rs index 67cefd98..30bd0a31 100644 --- a/tinycloud-node-server/src/storage/sealed.rs +++ b/tinycloud-node-server/src/storage/sealed.rs @@ -16,12 +16,21 @@ use crate::at_rest::AtRestRuntime; use crate::config::AtRestMode; use futures::io::{AsyncRead, AsyncReadExt}; -use std::{path::Path, sync::Arc}; +use std::{ + path::Path, + pin::Pin, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + task::{Context, Poll}, +}; use tinycloud_auth::resource::SpaceId; use tinycloud_core::{ at_rest::{ - space_tag, BlobIdentity, DataKey, DataKeyError, DataKeyProvider, DecryptReader, - EncryptReader, EnvelopeError, Header, ObjectNameKey, SegmentRange, HEADER_LEN, + keyring::AtRestError, space_tag, BlobIdentity, DataKey, DataKeyError, DataKeyProvider, + DecryptReader, EncryptReader, EnvelopeError, Header, ObjectNameKey, SegmentRange, + HEADER_LEN, }, hash::Hash, storage::{ByteRangeSpec, Content}, @@ -35,9 +44,16 @@ pub enum AtRestStoreError { /// once an operator acts: answer 503. #[error("encrypted storage is unavailable: {0}")] Unavailable(&'static str), - /// The key store could not be read (database or mirror). Answer 503. - #[error("at-rest key store failed: {0}")] - KeyStore(String), + /// The key store could not be read (database or mirror), or a key was + /// not prepared before the write lock. A reason code only; the backend + /// detail goes to the server log. Answer 503. + #[error("at-rest key store unavailable: {0}")] + KeyStore(&'static str), + /// The key store holds something an operator must look at (a stored data + /// key that does not unwrap, an inconsistent row). A reason code only. + /// Answer 500. + #[error("at-rest key store inconsistent: {0}")] + KeyInconsistent(&'static str), /// The sealed object failed authentication or does not match its header. #[error("stored object failed integrity checks: {0}")] Integrity(&'static str), @@ -48,6 +64,10 @@ pub enum AtRestStoreError { /// `strict` mode refuses legacy plaintext objects. #[error("legacy plaintext object refused in strict mode")] LegacyRefused, + /// A node fault unrelated to the stored bytes (a sealing task panicked). + /// Answer 500. + #[error("internal at-rest failure: {0}")] + Internal(&'static str), } impl AtRestStoreError { @@ -56,10 +76,70 @@ impl AtRestStoreError { use rocket::http::Status; match self { Self::Unavailable(_) | Self::KeyStore(_) => Status::ServiceUnavailable, - Self::Integrity(_) | Self::Unsupported(_) | Self::LegacyRefused => { - Status::InternalServerError + Self::KeyInconsistent(_) + | Self::Integrity(_) + | Self::Unsupported(_) + | Self::LegacyRefused + | Self::Internal(_) => Status::InternalServerError, + } + } + + /// Logs integrity and unsupported-format failures (typed, rate-limited, + /// no object names or space ids). Called where a request fails with it. + pub fn log_failure(&self) { + match self { + Self::Integrity(why) => log_integrity_failure("integrity", why), + Self::Unsupported(_) => log_integrity_failure("unsupported", "later format version"), + Self::KeyInconsistent(why) => log_integrity_failure("key_store", why), + _ => {} + } + } +} + +/// At most one log line per second for sealed-object integrity failures; the +/// number suppressed in between is reported with the next line. +fn log_integrity_failure(kind: &'static str, why: &'static str) { + static LAST_SECOND: AtomicU64 = AtomicU64::new(0); + static SUPPRESSED: AtomicU64 = AtomicU64::new(0); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + if LAST_SECOND.swap(now, Ordering::Relaxed) == now { + SUPPRESSED.fetch_add(1, Ordering::Relaxed); + return; + } + let suppressed = SUPPRESSED.swap(0, Ordering::Relaxed); + tracing::error!( + kind, + why, + suppressed, + "at-rest sealed block failed its checks" + ); +} + +/// A decrypting reader that logs (as [`AtRestStoreError::log_failure`] does) +/// when a sealed object fails authentication part way through a response. +pub struct LoggedDecrypt(DecryptReader); + +impl LoggedDecrypt { + pub fn new(inner: DecryptReader) -> Self { + Self(inner) + } +} + +impl AsyncRead for LoggedDecrypt { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let polled = Pin::new(&mut self.0).poll_read(cx, buf); + if let Poll::Ready(Err(e)) = &polled { + if e.kind() == std::io::ErrorKind::InvalidData { + log_integrity_failure("integrity", "authentication failed mid-stream"); } } + polled } } @@ -82,8 +162,32 @@ impl From for AtRestStoreError { fn from(e: DataKeyError) -> Self { match e { DataKeyError::Locked => Self::Unavailable("at-rest keys are locked"), + DataKeyError::NotPrepared => Self::KeyStore("data key not prepared"), DataKeyError::UnknownKey { .. } => Self::Integrity("header names an unknown data key"), - other => Self::KeyStore(other.to_string()), + DataKeyError::Backend(source) => { + let mapped = if source + .downcast_ref::() + .is_some() + { + Self::KeyStore("store_unavailable") + } else if let Some(e) = source.downcast_ref::() { + match e { + AtRestError::Store(_) => Self::KeyStore("store_unavailable"), + AtRestError::Mirror(_) => Self::KeyStore("mirror_unavailable"), + AtRestError::Locked | AtRestError::NotProvisioned => { + Self::Unavailable("at-rest keys are locked") + } + e if e.is_transient() => Self::KeyStore("key store busy"), + _ => Self::KeyInconsistent("inconsistent"), + } + } else { + // Unwrapping a stored data key failed. + Self::KeyInconsistent("stored data key does not authenticate") + }; + tracing::warn!(error = %source, reason = %mapped, "at-rest key store failure"); + mapped + } + _ => Self::KeyStore("key store failed"), } } } @@ -217,6 +321,46 @@ pub async fn prepare_write(view: &AtRestView, space: &SpaceId) -> Result<(), AtR Ok(()) } +/// Whether writing invocations must be authorized and their spaces prepared +/// before the write lock: whenever keys are set up (sealed blocks may then be +/// read or written, in any mode). +pub fn prepares(view: &AtRestView) -> bool { + view.provider.is_some() +} + +/// Gets a space ready for sealed reads before the invocation's database +/// write lock and transaction: puts back any data key the database lost, so +/// a read inside the transaction only looks keys up. Never creates a key. +pub async fn prepare_read(view: &AtRestView, space: &SpaceId) -> Result<(), AtRestStoreError> { + match &view.provider { + Some(provider) => Ok(provider.prepare_read(&space.to_string()).await?), + None => Ok(()), + } +} + +/// The name `contains` checks: the one a write would use (INTERFACES §7), +/// so an existing legacy copy does not count in `encrypt`/`strict`. +pub enum ContainsName { + Legacy, + Sealed(SealedName), +} + +pub fn contains_name( + view: &AtRestView, + space: &SpaceId, + hash: &Hash, +) -> Result { + if view.mode == AtRestMode::Off { + return Ok(ContainsName::Legacy); + } + let provider = view + .provider + .as_ref() + .ok_or(AtRestStoreError::Unavailable("at-rest keys are not set up"))?; + let nk = provider.object_name_key()?; + Ok(ContainsName::Sealed(SealedName::new(&nk, space, hash))) +} + /// Where a new block is written. pub enum WriteTarget { Legacy, @@ -435,7 +579,7 @@ pub async fn seal_vec( tinycloud_core::at_rest::seal(&key, &identity(&space, &hash), plaintext) }) .await - .map_err(|_| AtRestStoreError::Integrity("sealing task failed"))? + .map_err(|_| AtRestStoreError::Internal("sealing task failed"))? .map_err(Into::into) } diff --git a/tinycloud-node-server/src/storage/sealed/tests.rs b/tinycloud-node-server/src/storage/sealed/tests.rs index bd895c34..4691ae3e 100644 --- a/tinycloud-node-server/src/storage/sealed/tests.rs +++ b/tinycloud-node-server/src/storage/sealed/tests.rs @@ -356,39 +356,76 @@ fn space_tags_need_unlocked_keys() { ); } +/// At-rest causes answer 503 (retry may help) or 500 (operator must act) on +/// the KV GET routes and on `/invoke`, reads and writes alike; anything else +/// keeps its old mapping. #[test] -fn at_rest_causes_map_to_503_or_500_on_kv_reads() { +fn at_rest_causes_map_to_503_or_500_on_kv_reads_and_invoke() { use crate::storage::{file_system::FileSystemStoreError, s3::S3StoreError}; - use tinycloud_core::storage::either::EitherError; - let read = |e: AtRestStoreError, s3: bool| { - let store = if s3 { + use tinycloud_core::{storage::either::EitherError, TxStoreError}; + let store_err = |e: AtRestStoreError, s3: bool| { + if s3 { EitherError::A(S3StoreError::AtRest(e)) } else { EitherError::B(FileSystemStoreError::AtRest(e)) - }; - crate::routes::kv_read_failure(EitherError::B(store)).0 + } + }; + let cases = || { + [ + ( + AtRestStoreError::Unavailable("locked"), + Status::ServiceUnavailable, + ), + ( + AtRestStoreError::KeyStore("store_unavailable"), + Status::ServiceUnavailable, + ), + ( + AtRestStoreError::KeyInconsistent("inconsistent"), + Status::InternalServerError, + ), + ( + AtRestStoreError::Integrity("x"), + Status::InternalServerError, + ), + ( + AtRestStoreError::Unsupported(2), + Status::InternalServerError, + ), + (AtRestStoreError::LegacyRefused, Status::InternalServerError), + ( + AtRestStoreError::Internal("sealing task failed"), + Status::InternalServerError, + ), + ] }; for s3 in [true, false] { - assert_eq!( - read(AtRestStoreError::Unavailable("locked"), s3), - Status::ServiceUnavailable - ); - assert_eq!( - read(AtRestStoreError::KeyStore("pg".into()), s3), - Status::ServiceUnavailable - ); - assert_eq!( - read(AtRestStoreError::Integrity("x"), s3), - Status::InternalServerError - ); - assert_eq!( - read(AtRestStoreError::Unsupported(2), s3), - Status::InternalServerError - ); - assert_eq!( - read(AtRestStoreError::LegacyRefused, s3), - Status::InternalServerError - ); + for (e, want) in cases() { + let what = e.to_string(); + assert_eq!( + crate::routes::kv_read_failure(EitherError::B(store_err(e, s3))).0, + want, + "kv read: {what}" + ); + } + for (e, want) in cases() { + let what = e.to_string(); + let read = TxStoreError::StoreRead(store_err(e, s3)); + assert_eq!( + crate::routes::at_rest_invoke_status(&read), + Some(want), + "invoke read: {what}" + ); + } + for (e, want) in cases() { + let what = e.to_string(); + let write = TxStoreError::StoreWrite(store_err(e, s3)); + assert_eq!( + crate::routes::at_rest_invoke_status(&write), + Some(want), + "invoke write: {what}" + ); + } } let other = EitherError::B(EitherError::B(FileSystemStoreError::Io( std::io::Error::other("disk"), @@ -397,4 +434,29 @@ fn at_rest_causes_map_to_503_or_500_on_kv_reads() { crate::routes::kv_read_failure(other).0, Status::InternalServerError ); + let io = TxStoreError::StoreRead(EitherError::B(FileSystemStoreError::Io( + std::io::Error::other("disk"), + ))); + assert_eq!(crate::routes::at_rest_invoke_status(&io), None); +} + +/// Key-store failures answer with a reason code only; backend detail stays +/// in the server log. A stored key that does not unwrap is a 500. +#[test] +fn key_store_failures_carry_reason_codes_only() { + use tinycloud_core::sea_orm::DbErr; + let db = AtRestStoreError::from(DataKeyError::Backend(Box::new(DbErr::Custom( + "SELECT secret_table_detail".into(), + )))); + assert_eq!(db.status(), Status::ServiceUnavailable); + assert!(!db.to_string().contains("secret_table_detail"), "{db}"); + let unwrap = AtRestStoreError::from(DataKeyError::Backend(Box::new(std::io::Error::other( + "aead tag mismatch detail", + )))); + assert_eq!(unwrap.status(), Status::InternalServerError); + assert!(!unwrap.to_string().contains("detail"), "{unwrap}"); + assert_eq!( + AtRestStoreError::from(DataKeyError::NotPrepared).status(), + Status::ServiceUnavailable + ); } diff --git a/tinycloud-node-server/tests/at_rest_fs_node.rs b/tinycloud-node-server/tests/at_rest_fs_node.rs index 093cb457..54b2ac4d 100644 --- a/tinycloud-node-server/tests/at_rest_fs_node.rs +++ b/tinycloud-node-server/tests/at_rest_fs_node.rs @@ -121,7 +121,7 @@ async fn session(client: &Client) -> Result { "x": base64::encode_config(key.verifying_key().to_bytes(), base64::URL_SAFE_NO_PAD), }))?; let prepared = prepare_session(serde_json::from_value(json!({ - "abilities": {"kv": {"": ["tinycloud.kv/put", "tinycloud.kv/get"]}}, + "abilities": {"kv": {"": ["tinycloud.kv/put", "tinycloud.kv/get", "tinycloud.kv/del"]}}, "address": format!("0x{}", encode_eip55(&address)), "chainId": 1, "domain": "localhost", "issuedAt": rfc3339(now - time::Duration::minutes(1))?, @@ -144,13 +144,28 @@ async fn invoke( action: &str, body: &str, ) -> Result<(Status, String)> { - let resource = session.space_id.clone().to_resource( - "kv".parse::()?, - Some(path.parse::()?), - None, - None, - ); - let invocation = session.invoke_any([(resource, [action.parse::()?])], None)?; + invoke_in(client, session, &session.space_id, &[(path, action)], body).await +} + +/// One invocation with several `(path, action)` capabilities in `space`. +async fn invoke_in( + client: &Client, + session: &Session, + space: &SpaceId, + caps: &[(&str, &str)], + body: &str, +) -> Result<(Status, String)> { + let mut targets = Vec::new(); + for (path, action) in caps { + let resource = space.clone().to_resource( + "kv".parse::()?, + Some(path.parse::()?), + None, + None, + ); + targets.push((resource, [action.parse::()?])); + } + let invocation = session.invoke_any(targets, None)?; let response = client .post("/invoke") .header(Header::new("Authorization", invocation.encode()?)) @@ -164,6 +179,12 @@ async fn invoke( )) } +/// Rows in the node's `space_data_key` table. +fn data_key_rows(dir: &FsPath) -> Result { + let db = rusqlite::Connection::open(dir.join("caps.db"))?; + Ok(db.query_row("SELECT COUNT(*) FROM space_data_key", [], |r| r.get(0))?) +} + fn files(root: &FsPath) -> Vec { let mut out = Vec::new(); let mut stack = vec![root.to_path_buf()]; @@ -279,3 +300,92 @@ async fn an_off_node_still_stores_plaintext_under_legacy_names() -> Result<()> { ); Ok(()) } + +/// TC-887 S3 review B1: an invocation the node refuses (spaces it does not +/// host, no authority there) creates no data key and no mirror object. +#[tokio::test] +async fn a_refused_put_creates_no_data_key_or_mirror_object() -> Result<()> { + let dir = tempfile::tempdir()?; + let client = node(dir.path(), json!({"mode": "encrypt", "escrow_none": true})).await?; + let session = session(&client).await?; + let mirror = dir.path().join("at-rest"); + let (rows, mirrored) = (data_key_rows(dir.path())?, files(&mirror).len()); + let (_, address, _) = wallet()?; + for n in 0..3 { + let foreign: SpaceId = format!( + "tinycloud:pkh:eip155:1:0x{}:never-hosted-{n}", + encode_eip55(&address) + ) + .parse()?; + let (status, body) = invoke_in( + &client, + &session, + &foreign, + &[("notes/x", "tinycloud.kv/put")], + "value", + ) + .await?; + ensure!(status != Status::Ok, "put to {foreign}: {status} {body}"); + } + ensure!( + data_key_rows(dir.path())? == rows, + "refused puts created data keys" + ); + ensure!( + files(&mirror).len() == mirrored, + "refused puts created mirror objects" + ); + Ok(()) +} + +/// TC-887 S3 review B2: a mixed get + delete (the transaction path, under the +/// SQLite writer lock) of sealed values after the database lost the space's +/// data-key row (restored to before it; the mirror still holds it). The key +/// is put back before the lock, so the request completes instead of waiting +/// on the lock it holds. +#[tokio::test] +async fn a_mixed_read_and_delete_after_losing_the_data_key_row_completes() -> Result<()> { + let dir = tempfile::tempdir()?; + let at_rest = json!({"mode": "encrypt", "escrow_none": true}); + { + let client = node(dir.path(), at_rest.clone()).await?; + let session = session(&client).await?; + for (path, value) in values() { + let (status, body) = + invoke(&client, &session, &path, "tinycloud.kv/put", &value).await?; + ensure!(status == Status::Ok, "put {path}: {status} {body}"); + } + } + rusqlite::Connection::open(dir.path().join("caps.db"))? + .execute("DELETE FROM space_data_key", [])?; + + let client = node(dir.path(), at_rest).await?; + let session = session(&client).await?; + let (status, body) = tokio::time::timeout( + std::time::Duration::from_secs(30), + invoke_in( + &client, + &session, + &session.space_id, + &[ + ("notes/short", "tinycloud.kv/get"), + ("notes/big", "tinycloud.kv/del"), + ], + "", + ), + ) + .await + .context("mixed get + del hung (key restore under the writer lock?)")??; + ensure!( + status != Status::ServiceUnavailable && status != Status::InternalServerError, + "mixed get + del: {status} {body}" + ); + ensure!(data_key_rows(dir.path())? == 1, "the data key is put back"); + let (path, value) = values().remove(0); + let (status, body) = invoke(&client, &session, &path, "tinycloud.kv/get", "").await?; + ensure!( + status == Status::Ok && body == value, + "get {path}: {status}" + ); + Ok(()) +} From a6201d0d952813d01b87c1b8f7c5ac325b587c60 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 16:08:25 +0000 Subject: [PATCH 5/5] fix(TC-887): integrity logs name the sealed object; unauthorized hosted-space put test; changelog (S3 review deltas) --- CHANGELOG.md | 1 + tinycloud-node-server/src/routes/mod.rs | 10 +--- .../src/storage/file_system.rs | 29 ++++++++--- tinycloud-node-server/src/storage/sealed.rs | 48 ++++++++++++------- .../tests/at_rest_fs_node.rs | 47 ++++++++++++++++-- 5 files changed, 100 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3941a362..50a6bbc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Encryption at rest for KV blocks, FileSystem store (TC-887 S3, off by default). With `at_rest.mode = "encrypt"`/`"strict"` and keys set up, new KV blocks are sealed (TCE1) under anonymous names and served back as plaintext; legacy blocks stay readable in `encrypt`, are refused in `strict`, and a node with locked keys answers 503 for sealed data. **Behaviour change on nodes with at-rest keys only:** a writing invocation is now authorized (and its spaces checked as hosted) before any key work, so a put to a space the node does not host, or without authority there, answers **401 instead of 404**; nodes without keys are unchanged. See `docs/at-rest/INTERFACES.md` §7. - New migration `m20261017_000000_epoch_order_space_index` adds `idx_epoch_order_space_parent_child ON epoch_order (space, parent, child)` for the head-epoch anti-join that every write runs under its space lock (TC-1274). On a fresh database the build is instant. On PostgreSQL, if a valid index of that name and shape already exists, the migration issues no DDL and takes no lock on `epoch_order`. If the index of that name is INVALID or has a different shape, the boot fails, and nothing is applied, with an error naming the fix. On an existing database **without** the index, the migration builds it inside the boot's migration transaction. That holds a SHARE lock that blocks writes for the whole build, and the lock wait is bounded by the 30 s `lock_timeout` (TC-1176). **Runbook (PostgreSQL, before upgrading a node with history):** in autocommit, run `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_epoch_order_space_parent_child ON epoch_order (space, parent, child);`. Then confirm `SELECT indisvalid FROM pg_index WHERE indexrelid = 'idx_epoch_order_space_parent_child'::regclass;` returns `t`. If it returns `f`, run `DROP INDEX CONCURRENTLY idx_epoch_order_space_parent_child;` and build it again. `down` keeps the index. MySQL is covered only by the generated statement, because the full MySQL migration chain does not run (TC-1314). - Request events include validated/echoed UUID correlation, typed replacement flags, node-keyed target account/space pseudonyms and per-request database acquisition/execution timings and attempt counts. Driver/actor instrumentation includes failures and request-awaited audit batches; forwarding requires explicit first-party origin classification. A shared core `Pseudonymizer` preserves existing account/space hashes, adds the service/space-scoped database hash primitive for F6, and is derived once at startup for injection into consumers. See `docs/request-telemetry.md` for the wire contract, key custody, restart joinability and coverage/exclusions (TC-959 N2). - CACAO/SIWE delegation ingestion normalizes ECDSA recovery IDs 0/1 to 27/28 before hashing and persisting the canonical DAG-CBOR bytes (TC-1071). Equivalent low-s signatures now share one delegation CID, including the CID used for revocation lookup. High-s signatures are rejected during decoding, matching the existing EIP-191 verifier and EIP-2 low-s convention; UCAN/JWS bytes remain unchanged. Existing stored delegation IDs are not rewritten. diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index 71157c11..13e431e7 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -144,10 +144,7 @@ pub(crate) fn at_rest_invoke_status( ) -> Option { match e { TxStoreError::StoreRead(err) | TxStoreError::StoreWrite(err) => { - crate::storage::sealed::block_store_at_rest_error(err).map(|e| { - e.log_failure(); - e.status() - }) + crate::storage::sealed::block_store_at_rest_error(err).map(|e| e.status()) } _ => None, } @@ -163,10 +160,7 @@ pub(crate) fn kv_read_failure( use tinycloud_core::storage::either::EitherError; let status = match &e { EitherError::B(err) => crate::storage::sealed::block_store_at_rest_error(err) - .map(|e| { - e.log_failure(); - e.status() - }) + .map(|e| e.status()) .unwrap_or(Status::InternalServerError), EitherError::A(_) => Status::InternalServerError, }; diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index f72062d8..4ea57348 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -185,12 +185,17 @@ impl FileSystemStore { Err(e) => return Err(e.into()), }; let len = file.metadata().await?.len(); + let logged = |e: AtRestStoreError| { + e.log_at(name); + e + }; let (size, reader) = open_sealed(provider, space, id, file.take(len).compat(), len) - .await? + .await + .map_err(logged)? .into_inner(); Ok(Some(Content::new( size, - AsyncEither::Right(Box::new(LoggedDecrypt::new(reader))), + AsyncEither::Right(Box::new(LoggedDecrypt::new(reader, name))), ))) } @@ -273,11 +278,19 @@ impl FileSystemStore { Err(e) => return Err(e.into()), }; let len = file.metadata().await?.len(); + let logged = |e: AtRestStoreError| { + e.log_at(name); + e + }; let mut header = [0u8; HEADER_LEN]; - file.read_exact(&mut header) + file.read_exact(&mut header).await.map_err(|_| { + logged(AtRestStoreError::Integrity( + "object shorter than its header", + )) + })?; + let plan = plan_range(provider, space, &header, len, spec) .await - .map_err(|_| AtRestStoreError::Integrity("object shorter than its header"))?; - let plan = plan_range(provider, space, &header, len, spec).await?; + .map_err(logged)?; let Some(r) = plan.range else { return Ok(Some(RangeRead::Unsatisfiable { total_size: plan.plaintext_len(), @@ -285,14 +298,16 @@ impl FileSystemStore { }; file.seek(SeekFrom::Start(r.ciphertext_start())).await?; let span = r.ciphertext_end() - r.ciphertext_start() + 1; - let reader = plan.reader(space, id, r, file.take(span).compat())?; + let reader = plan + .reader(space, id, r, file.take(span).compat()) + .map_err(logged)?; let range = r.plaintext_range(); Ok(Some(RangeRead::Content { total_size: plan.plaintext_len(), range, content: Content::new( range.len(), - AsyncEither::Right(Box::new(LoggedDecrypt::new(reader))), + AsyncEither::Right(Box::new(LoggedDecrypt::new(reader, name))), ), })) } diff --git a/tinycloud-node-server/src/storage/sealed.rs b/tinycloud-node-server/src/storage/sealed.rs index 30bd0a31..f33faf48 100644 --- a/tinycloud-node-server/src/storage/sealed.rs +++ b/tinycloud-node-server/src/storage/sealed.rs @@ -84,21 +84,24 @@ impl AtRestStoreError { } } - /// Logs integrity and unsupported-format failures (typed, rate-limited, - /// no object names or space ids). Called where a request fails with it. - pub fn log_failure(&self) { - match self { - Self::Integrity(why) => log_integrity_failure("integrity", why), - Self::Unsupported(_) => log_integrity_failure("unsupported", "later format version"), - Self::KeyInconsistent(why) => log_integrity_failure("key_store", why), - _ => {} - } + /// Logs an integrity, unsupported-format or inconsistent-key failure of + /// the sealed object `name` (typed, rate-limited). The locator is the + /// sealed name `v1/{space tag}/{object tag}`: keyed hashes an operator can + /// find in the store, never a space id or content hash. + pub fn log_at(&self, name: &SealedName) { + let (kind, why) = match self { + Self::Integrity(why) => ("integrity", *why), + Self::Unsupported(_) => ("unsupported", "later format version"), + Self::KeyInconsistent(why) => ("key_store", *why), + _ => return, + }; + log_integrity_failure(kind, why, &name.key()); } } /// At most one log line per second for sealed-object integrity failures; the /// number suppressed in between is reported with the next line. -fn log_integrity_failure(kind: &'static str, why: &'static str) { +fn log_integrity_failure(kind: &'static str, why: &'static str, object: &str) { static LAST_SECOND: AtomicU64 = AtomicU64::new(0); static SUPPRESSED: AtomicU64 = AtomicU64::new(0); let now = std::time::SystemTime::now() @@ -112,18 +115,25 @@ fn log_integrity_failure(kind: &'static str, why: &'static str) { tracing::error!( kind, why, + object, suppressed, "at-rest sealed block failed its checks" ); } -/// A decrypting reader that logs (as [`AtRestStoreError::log_failure`] does) +/// A decrypting reader that logs (as [`AtRestStoreError::log_at`] does) /// when a sealed object fails authentication part way through a response. -pub struct LoggedDecrypt(DecryptReader); +pub struct LoggedDecrypt { + inner: DecryptReader, + object: String, +} impl LoggedDecrypt { - pub fn new(inner: DecryptReader) -> Self { - Self(inner) + pub fn new(inner: DecryptReader, name: &SealedName) -> Self { + Self { + inner, + object: name.key(), + } } } @@ -133,10 +143,15 @@ impl AsyncRead for LoggedDecrypt { cx: &mut Context<'_>, buf: &mut [u8], ) -> Poll> { - let polled = Pin::new(&mut self.0).poll_read(cx, buf); + let this = &mut *self; + let polled = Pin::new(&mut this.inner).poll_read(cx, buf); if let Poll::Ready(Err(e)) = &polled { if e.kind() == std::io::ErrorKind::InvalidData { - log_integrity_failure("integrity", "authentication failed mid-stream"); + log_integrity_failure( + "integrity", + "authentication failed mid-stream", + &this.object, + ); } } polled @@ -177,6 +192,7 @@ impl From for AtRestStoreError { AtRestError::Locked | AtRestError::NotProvisioned => { Self::Unavailable("at-rest keys are locked") } + AtRestError::DekUnavailable(why) => Self::KeyStore(why), e if e.is_transient() => Self::KeyStore("key store busy"), _ => Self::KeyInconsistent("inconsistent"), } diff --git a/tinycloud-node-server/tests/at_rest_fs_node.rs b/tinycloud-node-server/tests/at_rest_fs_node.rs index 54b2ac4d..699720be 100644 --- a/tinycloud-node-server/tests/at_rest_fs_node.rs +++ b/tinycloud-node-server/tests/at_rest_fs_node.rs @@ -92,8 +92,19 @@ async fn delegate(client: &Client, headers: &impl serde::Serialize) -> Result<() Ok(()) } -/// Host delegation for the space, then a KV session. +/// Host delegation for the space, then a KV session with all KV abilities. async fn session(client: &Client) -> Result { + session_with( + client, + &["tinycloud.kv/put", "tinycloud.kv/get", "tinycloud.kv/del"], + 114, + ) + .await +} + +/// Host delegation for the space, then a KV session with exactly `abilities` +/// under the session key seeded by `key_seed`. +async fn session_with(client: &Client, abilities: &[&str], key_seed: u8) -> Result { let (wallet, address, space) = wallet()?; let now = time::OffsetDateTime::now_utc(); let peer = client @@ -114,19 +125,19 @@ async fn session(client: &Client) -> Result { }))?; delegate(client, &siwe_to_delegation_headers(host)).await?; - let key = ed25519_dalek::SigningKey::from_bytes(&[114; 32]); + let key = ed25519_dalek::SigningKey::from_bytes(&[key_seed; 32]); let jwk: JWK = serde_json::from_value(json!({ "kty": "OKP", "crv": "Ed25519", "d": base64::encode_config(key.to_bytes(), base64::URL_SAFE_NO_PAD), "x": base64::encode_config(key.verifying_key().to_bytes(), base64::URL_SAFE_NO_PAD), }))?; let prepared = prepare_session(serde_json::from_value(json!({ - "abilities": {"kv": {"": ["tinycloud.kv/put", "tinycloud.kv/get", "tinycloud.kv/del"]}}, + "abilities": {"kv": {"": abilities}}, "address": format!("0x{}", encode_eip55(&address)), "chainId": 1, "domain": "localhost", "issuedAt": rfc3339(now - time::Duration::minutes(1))?, "expirationTime": rfc3339(now + time::Duration::hours(1))?, - "spaceId": space, "jwk": jwk, "nonce": "tc887s3bnode", + "spaceId": space, "jwk": jwk, "nonce": format!("tc887s3bnode{key_seed}"), }))?)?; let signature = sign(&wallet, &prepared.siwe)?; let session = complete_session_setup(SignedSession { @@ -338,6 +349,34 @@ async fn a_refused_put_creates_no_data_key_or_mirror_object() -> Result<()> { Ok(()) } +/// TC-887 S3 review B1/R1: a put to a space the node hosts, by a session that +/// holds no `kv/put` there, is refused by authorization before any key work: +/// no data key and no mirror object for the space. +#[tokio::test] +async fn an_unauthorized_put_to_a_hosted_space_creates_no_data_key() -> Result<()> { + let dir = tempfile::tempdir()?; + let client = node(dir.path(), json!({"mode": "encrypt", "escrow_none": true})).await?; + let reader = session_with(&client, &["tinycloud.kv/get"], 115).await?; + let mirror = dir.path().join("at-rest"); + let (rows, mirrored) = (data_key_rows(dir.path())?, files(&mirror).len()); + for path in ["notes/a", "notes/b", "notes/c"] { + let (status, body) = invoke(&client, &reader, path, "tinycloud.kv/put", "value").await?; + ensure!( + status == Status::Unauthorized || status == Status::Forbidden, + "unauthorized put {path}: {status} {body}" + ); + } + ensure!( + data_key_rows(dir.path())? == rows, + "an unauthorized put created a data key" + ); + ensure!( + files(&mirror).len() == mirrored, + "an unauthorized put created a mirror object" + ); + Ok(()) +} + /// TC-887 S3 review B2: a mixed get + delete (the transaction path, under the /// SQLite writer lock) of sealed values after the database lost the space's /// data-key row (restored to before it; the mirror still holds it). The key