From 5f34b4d372108e2619e467cea71c41c117c3db0a Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 14:37:31 +0000 Subject: [PATCH 01/17] TC-735: Persist KV blobs before sequence lock --- CHANGELOG.md | 1 + docs/kv-sync.md | 28 + tinycloud-core/src/db.rs | 524 +++++++++------- tinycloud-core/src/kv_orphans.rs | 593 ++++++++++++++++++ tinycloud-core/src/kv_sync.rs | 298 ++++++++- tinycloud-core/src/lib.rs | 1 + .../m20261015_020000_kv_orphan_candidates.rs | 35 ++ tinycloud-core/src/migrations/mod.rs | 2 + .../src/models/kv_orphan_candidate.rs | 19 + tinycloud-core/src/models/mod.rs | 1 + tinycloud-node-server/src/lib.rs | 46 +- .../src/storage/file_system.rs | 149 +++-- tinycloud-node-server/src/storage/s3.rs | 156 +++-- .../tests/postgres_kv_write_throughput.rs | 118 +++- 14 files changed, 1629 insertions(+), 342 deletions(-) create mode 100644 tinycloud-core/src/kv_orphans.rs create mode 100644 tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs create mode 100644 tinycloud-core/src/models/kv_orphan_candidate.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 18907086..60147722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- KV puts persist content-addressed blobs before taking the per-space sequence lock, while the lock still orders committed `kv/sync` cursors. A durable hash-fenced candidate table (`m20261015_020000_kv_orphan_candidates`) supports immediate cleanup and physical-byte quota refunds after known failures; a 60-second collector reclaims crash leftovers after a five-minute grace. Quota remains conservative and per process, not a durable global ledger (TC-735). - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/docs/kv-sync.md b/docs/kv-sync.md index 7ed519bf..d09fc408 100644 --- a/docs/kv-sync.md +++ b/docs/kv-sync.md @@ -129,3 +129,31 @@ cursor. Read audit records every poll. Poll on demand, or no more often than every 60 seconds. + +## Write ordering and blob recovery + +KV puts stage and authorize content before upload. The node registers a +durable `(space, content hash)` recovery candidate, then persists the blob +before taking the per-space sequence lock. The database transaction rechecks +authorization and conditional-write preconditions under that lock; it holds +the lock through commit, so `seq` and sync cursors remain commit-ordered even +when same-space uploads finish out of order. An unauthorized write uploads +nothing. + +A failed upload, precondition, or database write triggers an immediate +hash-fenced reference check. Unreferenced blobs are removed and their +physical-byte quota is refunded before the failure returns. Historical +`kv_write` references are retained as well as current values, because a +client can still read an older value. The candidate row is removed with a +successful KV commit; a failed cleanup leaves it for the collector. After a +crash, the node scans candidates at 60-second intervals, collecting those +older than five minutes in bounded batches of 256 until the eligible backlog +is drained. S3 and filesystem stores reserve bytes before a potentially +ambiguous write; deleting the resulting object or canceling a reservation +for an object that never materialized refunds once, not twice. + +Quota is conservative physical accounting, **not commit-only accounting**: +in-flight uploads and crash leftovers may temporarily count against it. +`SpaceSizes` remains in-memory and per process; writes or deletions on other +nodes do not synchronize that counter. Globally exact multi-node accounting +requires a separate shared byte ledger (TC-1162). diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index d1e6d8c0..defe7e0e 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -12,8 +12,8 @@ use crate::read_audit::ReadAuditPipeline; use crate::relationships::*; use crate::sql_sizes::SqlSizes; use crate::storage::{ - either::EitherError, ByteRangeSpec, Content, HashBuffer, ImmutableReadStore, ImmutableStaging, - ImmutableWriteStore, RangeRead, StorageSetup, StoreSize, + either::EitherError, ByteRangeSpec, Content, HashBuffer, ImmutableDeleteStore, + ImmutableReadStore, ImmutableStaging, ImmutableWriteStore, RangeRead, StorageSetup, StoreSize, }; use crate::types::{ AccountDelegationRecord, CapabilitiesReadParams, DelegationQuery, DelegationQueryDirection, @@ -1481,7 +1481,7 @@ where precondition: Option, ) -> Result> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1505,7 +1505,7 @@ where precondition: Option, ) -> Result> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1529,7 +1529,7 @@ where path: Path, ) -> Result<(), TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1563,7 +1563,7 @@ where mode: InvokeMode, ) -> Result, TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1847,7 +1847,7 @@ where inputs: InvocationInputs, ) -> Result<(TransactResult, Vec>), TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1868,7 +1868,7 @@ where inputs: InvocationInputs, ) -> Result<(TransactResult, Vec>), TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1883,7 +1883,7 @@ where options: KvInvokeOptions, ) -> Result<(TransactResult, Vec>), TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1903,7 +1903,7 @@ where options: KvInvokeOptions, ) -> Result<(TransactResult, Vec>), TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -1919,7 +1919,7 @@ where mode: InvokeMode, ) -> Result<(TransactResult, Vec>), TxStoreError> where - B: ImmutableWriteStore + ImmutableReadStore, + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, S: ImmutableStaging, S::Writable: 'static + Unpin, { @@ -2086,241 +2086,301 @@ where } } - // TC-732: conditional writes no longer need SERIALIZABLE. The - // per-space sequence lock is taken right after BEGIN, before the - // precondition read, so under READ COMMITTED that read already sees - // every earlier same-space commit and no same-space writer can commit - // until this one does. A failed precondition is a 412, never a false - // success (see `postgres_conditional_put_is_atomic_under_concurrency`). - let sequence_spaces = invocation.0.spaces().cloned().collect::>(); - let _sequence_guards = self.acquire_space_sequence_guards(&sequence_spaces).await; - let _writer = match &self.writer_lock { - Some(lock) => Some(lock.lock().await), - None => None, - }; - let begin_start = Instant::now(); - let tx_result = self.begin_graph_transaction().await; - crate::telemetry::observe_stage( - crate::telemetry::InvocationStage::DbTxBegin, - crate::telemetry::StageOutcome::from(tx_result.is_ok()), - begin_start.elapsed(), - ); - let tx = tx_result?; - // TC-732: order this transaction's event sequence under the - // per-space lock. On PostgreSQL this is an advisory lock, so the - // meeting legacy guard lock below is still the first row lock. - crate::kv_sync::lock_space_sequences(&tx, &sequence_spaces).await?; - // On SQLite `begin_graph_transaction` already holds the write lock, - // so no read-to-write upgrade race remains. Distinct spaces are - // locked in a stable order to prevent cross-space deadlocks on PG. - let mut legacy_spaces: Vec<_> = mutation_keys - .iter() - .filter(|(_, path)| crate::meeting_legacy_guard::protects(path.as_str())) - .map(|(space, _)| space) - .collect(); - legacy_spaces.sort_by_key(|space| space.to_string()); - legacy_spaces.dedup(); - for space in legacy_spaces { - if crate::meeting_legacy_guard::lock_writer(&tx, space).await? { - return Err(TxStoreError::LegacyMeetingFrozen); + // Check the entire invocation before creating an object in the block + // store. The transaction below repeats authorization after acquiring + // the sequence lock, preserving revocation and time ordering. + let now = OffsetDateTime::now_utc(); + match mode { + InvokeMode::Admitted => { + invocation::authorize_admitted(&self.conn, &invocation.0, now, None) + .await + .map_err(TxError::::from)?; } - } - // DbTxBody spans post-begin to pre-commit. The guard defaults to an - // `error` outcome so any `?`/early return inside the transaction is - // recorded as a failure; it is disarmed to `ok` right before commit. - let tx_body_timer = - crate::telemetry::StageTimer::start(crate::telemetry::InvocationStage::DbTxBody); - let auth_graph = match crate::auth_graph::AuthGraphSnapshot::load(&tx, &roots).await { - Ok(graph) => graph, - Err(error) => { - crate::telemetry::observe_stage( - crate::telemetry::InvocationStage::AuthorizationGraphLoad, - crate::telemetry::StageOutcome::Error, - authz_start.elapsed(), - ); - return Err(TxStoreError::Tx(match error { - revocation::ChainTraversalError::Db(error) => TxError::Db(error), - revocation::ChainTraversalError::LimitExceeded => { - TxError::ChainTraversalLimitExceeded - } - })); + InvokeMode::Public => { + invocation::verify_and_authorize(&self.conn, &invocation.0, now) + .await + .map_err(TxError::::from)?; } - }; - crate::telemetry::observe_stage( - crate::telemetry::InvocationStage::AuthorizationGraphLoad, - crate::telemetry::StageOutcome::Ok, - authz_start.elapsed(), - ); - let mut deleted_hashes = HashMap::new(); - for key @ (space, path) in &mutation_keys { - let current = get_kv_entity(&tx, space, path).await?; - if let Some(precondition) = options.preconditions.get(key) { - let matches = kv_precondition_matches( - *precondition, - current.as_ref().map(|entry| entry.value), - ); - if !matches { - return Err(TxStoreError::KvPreconditionFailed); - } + InvokeMode::Internal | InvokeMode::MeetingPublication => {} + } + let mut sequence_spaces = invocation.0.spaces().cloned().collect::>(); + sequence_spaces.sort_unstable(); + sequence_spaces.dedup(); + let hosted = space::Entity::find() + .filter(space::Column::Id.is_in(sequence_spaces.iter().cloned().map(SpaceIdWrap))) + .count(&self.conn) + .await?; + if hosted != sequence_spaces.len() as u64 { + return Err(TxError::SpaceNotFound.into()); + } + let mut blobs = write_hashes + .iter() + .map(|((space, _), hash)| (space.clone(), *hash)) + .collect::>(); + blobs.sort_unstable(); + blobs.dedup(); + if !blobs.is_empty() { + crate::kv_orphans::register_candidates(&self.conn, &blobs).await?; + } + // The database transaction fences the blob hash against a collector + // (and another writer of the same content) through commit or rollback. + // PostgreSQL only takes the per-space advisory sequence lock *after* + // all S3 uploads. SQLite has one writer transaction already. + let result = async { + let _writer = match &self.writer_lock { + Some(lock) => Some(lock.lock().await), + None => None, + }; + let begin_start = Instant::now(); + let tx_result = self.begin_graph_transaction().await; + crate::telemetry::observe_stage( + crate::telemetry::InvocationStage::DbTxBegin, + crate::telemetry::StageOutcome::from(tx_result.is_ok()), + begin_start.elapsed(), + ); + let tx = tx_result?; + if !blobs.is_empty() { + crate::kv_orphans::lock_candidates(&tx, &blobs).await?; } - if let Some(entry) = current { - deleted_hashes.insert(key.clone(), entry.value); + for ((space, path), stage) in stages.drain() { + let hash = self + .storage + .persist(&space, stage) + .await + .map_err(TxStoreError::StoreWrite)?; + debug_assert_eq!(write_hashes.get(&(space, path)), Some(&hash)); } - } - let caps = invocation.0.capabilities.clone(); - let invoker = invocation.0.invoker.clone(); - // Extract capabilities read params from UCAN facts field - // Facts is Vec, we look for an object with capabilitiesReadParams key - let caps_read_params: Option = invocation - .0 - .invocation - .payload() - .facts - .as_ref() - .and_then(|facts| { - facts.iter().find_map(|fact| { - fact.as_object() - .and_then(|obj| obj.get("capabilitiesReadParams")) - .and_then(|v| serde_json::from_value(v.clone()).ok()) - }) - }); - // verify and commit invocation and kv operations - let event = match mode { - InvokeMode::Internal | InvokeMode::MeetingPublication => { - Event::InternalInvocation(Box::new(invocation), ops) + // Only the database section assigns sequence numbers. Its advisory + // lock remains held through commit so kv/sync cursors stay ordered. + let _sequence_guards = self.acquire_space_sequence_guards(&sequence_spaces).await; + crate::kv_sync::lock_space_sequences(&tx, &sequence_spaces).await?; + // The sequence lock precedes every precondition read under READ + // COMMITTED, including when an upload was prepared concurrently. + // SQLite already holds its single-writer lock at BEGIN, preventing a + // read-to-write upgrade race. Multi-space PG locks are acquired in + // stable order to avoid cross-space deadlocks. + let mut legacy_spaces: Vec<_> = mutation_keys + .iter() + .filter(|(_, path)| crate::meeting_legacy_guard::protects(path.as_str())) + .map(|(space, _)| space) + .collect(); + legacy_spaces.sort_by_key(|space| space.to_string()); + legacy_spaces.dedup(); + for space in legacy_spaces { + if crate::meeting_legacy_guard::lock_writer(&tx, space).await? { + return Err(TxStoreError::LegacyMeetingFrozen); + } } - InvokeMode::Admitted => Event::AdmittedInvocation(Box::new(invocation), ops), - InvokeMode::Public => Event::Invocation(Box::new(invocation), ops), - }; - let commit = transact( - &tx, - &self.storage, - &self.secrets, - vec![event], - self.encryption.as_ref(), - Some(&auth_graph), - // Taken on this `tx` right after BEGIN, above. - &sequence_spaces, - ) - .await?; - - let mut results = Vec::new(); - // perform and record side effects - for cap in caps.iter().filter_map(|c| { - c.resource.tinycloud_resource().and_then(|r| { - Some(( - r.space(), - r.service().as_str(), - // TC-119: resolve deprecated aliases to canonical (see the - // staging loop above) — identity for canonical URNs. - crate::policy_capability::resolve_alias(c.ability.as_ref().as_ref()), - r.path()?, - )) - }) - }) { - match cap { - (space, "kv", "tinycloud.kv/get", path) => { - let data = - get_kv(&tx, &self.storage, space, path) - .await - .map_err(|e| match e { - EitherError::A(e) => TxStoreError::Tx(e.into()), - EitherError::B(e) => TxStoreError::StoreRead(e), - })?; - if let (Some(limit), Some((_, _, content))) = - (options.max_response_bytes, data.as_ref()) - { - if content.len() > limit { - return Err(TxStoreError::KvResponseTooLarge { - size: content.len(), - limit, - }); + // DbTxBody spans post-begin to pre-commit. The guard defaults to an + // `error` outcome so any `?`/early return inside the transaction is + // recorded as a failure; it is disarmed to `ok` right before commit. + let tx_body_timer = + crate::telemetry::StageTimer::start(crate::telemetry::InvocationStage::DbTxBody); + let auth_graph = match crate::auth_graph::AuthGraphSnapshot::load(&tx, &roots).await { + Ok(graph) => graph, + Err(error) => { + crate::telemetry::observe_stage( + crate::telemetry::InvocationStage::AuthorizationGraphLoad, + crate::telemetry::StageOutcome::Error, + authz_start.elapsed(), + ); + return Err(TxStoreError::Tx(match error { + revocation::ChainTraversalError::Db(error) => TxError::Db(error), + revocation::ChainTraversalError::LimitExceeded => { + TxError::ChainTraversalLimitExceeded } + })); + } + }; + crate::telemetry::observe_stage( + crate::telemetry::InvocationStage::AuthorizationGraphLoad, + crate::telemetry::StageOutcome::Ok, + authz_start.elapsed(), + ); + let mut deleted_hashes = HashMap::new(); + for key @ (space, path) in &mutation_keys { + let current = get_kv_entity(&tx, space, path).await?; + if let Some(precondition) = options.preconditions.get(key) { + let matches = kv_precondition_matches( + *precondition, + current.as_ref().map(|entry| entry.value), + ); + if !matches { + return Err(TxStoreError::KvPreconditionFailed); } - results.push(InvocationOutcome::KvRead(data)); } - (space, "kv", "tinycloud.kv/list", path) => { - let (list, truncated) = list_bounded_after( - &tx, - space, - path, - options.list_limit, - options.list_cursor.as_ref(), - ) - .await?; - results.push(InvocationOutcome::KvList(list, truncated, None)) + if let Some(entry) = current { + deleted_hashes.insert(key.clone(), entry.value); } - (space, "kv", "tinycloud.kv/del", path) => { - // KV deletion is logical. Blobs are content-addressed and may be - // shared by live sibling keys or retained version history. - results.push(InvocationOutcome::KvDelete( - deleted_hashes.get(&(space.clone(), path.clone())).copied(), - )) + } + let caps = invocation.0.capabilities.clone(); + let invoker = invocation.0.invoker.clone(); + // Extract capabilities read params from UCAN facts field + // Facts is Vec, we look for an object with capabilitiesReadParams key + let caps_read_params: Option = invocation + .0 + .invocation + .payload() + .facts + .as_ref() + .and_then(|facts| { + facts.iter().find_map(|fact| { + fact.as_object() + .and_then(|obj| obj.get("capabilitiesReadParams")) + .and_then(|v| serde_json::from_value(v.clone()).ok()) + }) + }); + // verify and commit invocation and kv operations + let event = match mode { + InvokeMode::Internal | InvokeMode::MeetingPublication => { + Event::InternalInvocation(Box::new(invocation), ops) } - (space, "kv", "tinycloud.kv/put", path) => { - if let Some(stage) = stages.remove(&(space.clone(), path.clone())) { - self.storage - .persist(space, stage) - .await - .map_err(TxStoreError::StoreWrite)?; + InvokeMode::Admitted => Event::AdmittedInvocation(Box::new(invocation), ops), + InvokeMode::Public => Event::Invocation(Box::new(invocation), ops), + }; + let commit = transact( + &tx, + &self.storage, + &self.secrets, + vec![event], + self.encryption.as_ref(), + Some(&auth_graph), + // Taken on this `tx` right after BEGIN, above. + &sequence_spaces, + ) + .await?; + + let mut results = Vec::new(); + // perform and record side effects + for cap in caps.iter().filter_map(|c| { + c.resource.tinycloud_resource().and_then(|r| { + Some(( + r.space(), + r.service().as_str(), + // TC-119: resolve deprecated aliases to canonical (see the + // staging loop above) — identity for canonical URNs. + crate::policy_capability::resolve_alias(c.ability.as_ref().as_ref()), + r.path()?, + )) + }) + }) { + match cap { + (space, "kv", "tinycloud.kv/get", path) => { + let data = + get_kv(&tx, &self.storage, space, path) + .await + .map_err(|e| match e { + EitherError::A(e) => TxStoreError::Tx(e.into()), + EitherError::B(e) => TxStoreError::StoreRead(e), + })?; + if let (Some(limit), Some((_, _, content))) = + (options.max_response_bytes, data.as_ref()) + { + if content.len() > limit { + return Err(TxStoreError::KvResponseTooLarge { + size: content.len(), + limit, + }); + } + } + results.push(InvocationOutcome::KvRead(data)); + } + (space, "kv", "tinycloud.kv/list", path) => { + let (list, truncated) = list_bounded_after( + &tx, + space, + path, + options.list_limit, + options.list_cursor.as_ref(), + ) + .await?; + results.push(InvocationOutcome::KvList(list, truncated, None)) + } + (space, "kv", "tinycloud.kv/del", path) => { + // KV deletion is logical. Blobs are content-addressed and may be + // shared by live sibling keys or retained version history. + results.push(InvocationOutcome::KvDelete( + deleted_hashes.get(&(space.clone(), path.clone())).copied(), + )) + } + (space, "kv", "tinycloud.kv/put", path) => { let hash = write_hashes .get(&(space.clone(), path.clone())) .copied() .expect("staged KV writes have a content hash"); - results.push(InvocationOutcome::KvWrite(hash)) + results.push(InvocationOutcome::KvWrite(hash)); } - } - (space, "kv", "tinycloud.kv/metadata", path) => results.push( - InvocationOutcome::KvMetadata(metadata_with_hash(&tx, space, path).await?), - ), - (space, "capabilities", "tinycloud.capabilities/read", path) - if path.as_str() == "all" => - { - match &caps_read_params { - None => { - // Backward compatible: no params means return all valid delegations - results.push(InvocationOutcome::OpenSessions( - get_valid_delegations(&tx, space, self.encryption.as_ref()).await?, - )) - } - Some(CapabilitiesReadParams::List { filters }) => { - // List with optional filters - results.push(InvocationOutcome::OpenSessions( - get_filtered_delegations( - &tx, - space, - &invoker, - filters.as_ref(), - self.encryption.as_ref(), - ) - .await?, - )) - } - Some(CapabilitiesReadParams::Chain { delegation_cid }) => { - // Get the delegation chain for a specific delegation - results.push(InvocationOutcome::DelegationChain( - get_delegation_chain( - &tx, - space, - delegation_cid, - self.encryption.as_ref(), - ) - .await?, - )) + (space, "kv", "tinycloud.kv/metadata", path) => results.push( + InvocationOutcome::KvMetadata(metadata_with_hash(&tx, space, path).await?), + ), + (space, "capabilities", "tinycloud.capabilities/read", path) + if path.as_str() == "all" => + { + match &caps_read_params { + None => { + // Backward compatible: no params means return all valid delegations + results.push(InvocationOutcome::OpenSessions( + get_valid_delegations(&tx, space, self.encryption.as_ref()) + .await?, + )) + } + Some(CapabilitiesReadParams::List { filters }) => { + // List with optional filters + results.push(InvocationOutcome::OpenSessions( + get_filtered_delegations( + &tx, + space, + &invoker, + filters.as_ref(), + self.encryption.as_ref(), + ) + .await?, + )) + } + Some(CapabilitiesReadParams::Chain { delegation_cid }) => { + // Get the delegation chain for a specific delegation + results.push(InvocationOutcome::DelegationChain( + get_delegation_chain( + &tx, + space, + delegation_cid, + self.encryption.as_ref(), + ) + .await?, + )) + } } } - } - _ => {} - }; - } + _ => {} + }; + } + if !blobs.is_empty() { + crate::kv_orphans::finish_candidates(&tx, &blobs).await?; + } - // Record the transaction body as successful (post-begin to - // pre-commit) before the commit itself; commit latency is tracked - // separately (EpochPersist on the delegate/revoke path). - tx_body_timer.observe_ok(); - // commit tx if all side effects worked - tx.commit().await?; - Ok((commit, results)) + // Record the transaction body as successful (post-begin to + // pre-commit) before the commit itself; commit latency is tracked + // separately (EpochPersist on the delegate/revoke path). + tx_body_timer.observe_ok(); + // commit tx if all side effects worked + tx.commit().await?; + Ok((commit, results)) + } + .await; + if result.is_err() && !blobs.is_empty() { + // The failed transaction (including any hash locks) has ended. + // A commit error may be outcome-unknown: the candidate fence and + // committed-reference check distinguish that from an orphan. + if let Err(error) = + crate::kv_orphans::refund_failed(&self.conn, &self.storage, &blobs).await + { + tracing::warn!( + ?error, + "failed to refund orphaned KV blob; periodic collector will retry" + ); + } + } + result } /// Execute a successful non-mutating invocation without entering the @@ -5073,6 +5133,14 @@ mod test { self.inner.persist(space, stage).await } } + #[async_trait::async_trait] + impl ImmutableDeleteStore for LegacyPausedStore { + type Error = std::io::Error; + + async fn remove(&self, space: &SpaceId, hash: &Hash) -> Result, Self::Error> { + self.inner.remove(space, hash).await + } + } #[tokio::test] async fn legacy_freeze_ack_waits_for_actual_in_flight_kv_storage_and_commit() { use crate::storage::memory::MemoryStaging; diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs new file mode 100644 index 00000000..7cec7935 --- /dev/null +++ b/tinycloud-core/src/kv_orphans.rs @@ -0,0 +1,593 @@ +//! Durable fences for blobs uploaded before a KV transaction's sequence lock. + +use std::{borrow::Cow, error::Error as StdError}; + +use sea_orm::{ + sea_query::{Expr, LockBehavior, LockType, OnConflict}, + ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, QueryFilter, + QueryOrder, QuerySelect, TransactionTrait, +}; +use time::{Duration, OffsetDateTime}; +use tinycloud_auth::resource::SpaceId; + +use crate::hash::Hash; +use crate::models::{current_kv, kv_orphan_candidate as candidate, kv_write}; +use crate::storage::ImmutableDeleteStore; +use crate::types::SpaceIdWrap; + +const MAX_BATCH_SIZE: u64 = 256; + +fn ordered_unique(blobs: &[(SpaceId, Hash)]) -> Cow<'_, [(SpaceId, Hash)]> { + if blobs.windows(2).all(|pair| pair[0] < pair[1]) { + return Cow::Borrowed(blobs); + } + let mut ordered = blobs.to_vec(); + ordered.sort_unstable(); + ordered.dedup(); + Cow::Owned(ordered) +} + +/// Durably register every possible orphan before uploading any of these blobs. +/// +/// An existing candidate gets a new grace period. This operation is intentionally +/// outside the KV transaction so a failed precondition or a crash leaves a +/// collectable candidate behind. +pub async fn register_candidates( + conn: &C, + blobs: &[(SpaceId, Hash)], +) -> Result<(), DbErr> { + for (space, value) in ordered_unique(blobs).iter() { + candidate::Entity::insert(candidate::ActiveModel { + space: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), + value: sea_orm::ActiveValue::Set(*value), + created_at: sea_orm::ActiveValue::Set(OffsetDateTime::now_utc()), + }) + .on_conflict( + OnConflict::columns([candidate::Column::Space, candidate::Column::Value]) + .update_column(candidate::Column::CreatedAt) + .to_owned(), + ) + .exec(conn) + .await?; + } + Ok(()) +} + +/// Fence each blob against concurrent collection before uploading it. +/// +/// Call after registration, inside the KV transaction, before any PUT or +/// per-space sequence lock. Hold the transaction through the upload and +/// successful KV commit. Sorting prevents reversed hash-lock deadlocks. +pub async fn lock_candidates( + tx: &DatabaseTransaction, + blobs: &[(SpaceId, Hash)], +) -> Result<(), DbErr> { + // Upsert again inside the transaction: a previous writer may have + // committed and deleted our pre-registered row while we waited to begin + // this transaction. The upsert owns the hash fence through commit on + // PostgreSQL, MySQL and SQLite, even when the row was absent. + register_candidates(tx, blobs).await +} + +/// Remove successful candidates atomically with their KV writes. +/// +/// Call only once the writes have succeeded and before committing the KV +/// transaction. Rollback leaves candidates for the collector. +pub async fn finish_candidates( + tx: &DatabaseTransaction, + blobs: &[(SpaceId, Hash)], +) -> Result<(), DbErr> { + for (space, value) in ordered_unique(blobs).iter() { + candidate::Entity::delete_by_id((SpaceIdWrap(space.clone()), *value)) + .exec(tx) + .await?; + } + Ok(()) +} + +/// A database operation or object-store deletion failed; the candidate is +/// retained for a future collection pass on rollback. +#[derive(Debug, thiserror::Error)] +pub enum CollectError { + #[error("orphan candidate database operation failed: {0}")] + Database(#[from] DbErr), + #[error("orphan blob deletion failed: {0}")] + Store(E), +} + +async fn remove_if_unreferenced( + tx: &DatabaseTransaction, + store: &S, + row: candidate::Model, +) -> Result> { + let historical = kv_write::Entity::find() + .filter(kv_write::Column::Space.eq(row.space.clone())) + .filter(kv_write::Column::Value.eq(row.value)) + .select_only() + .column(kv_write::Column::Seq); + let current = current_kv::Entity::find() + .filter(current_kv::Column::Space.eq(row.space.clone())) + .filter(current_kv::Column::Value.eq(row.value)) + .select_only() + .column(current_kv::Column::Seq); + // MySQL's REPEATABLE READ snapshot may precede the writer whose hash + // fence we just acquired. Locking reads see its committed references. + let (historical, current) = if tx.get_database_backend() == DbBackend::MySql { + (historical.lock_exclusive(), current.lock_exclusive()) + } else { + (historical, current) + }; + let referenced = historical.into_tuple::().one(tx).await?.is_some() + || current.into_tuple::().one(tx).await?.is_some(); + let removed = if referenced { + false + } else { + store + .remove(&row.space.0, &row.value) + .await + .map_err(CollectError::Store)? + .is_some() + }; + candidate::Entity::delete_by_id((row.space, row.value)) + .exec(tx) + .await?; + Ok(removed) +} + +/// Refund a known failed invocation before returning to its caller. The +/// durable candidate remains for periodic recovery if deletion fails. +pub async fn refund_failed( + conn: &C, + store: &S, + blobs: &[(SpaceId, Hash)], +) -> Result<(), CollectError> +where + C: ConnectionTrait + TransactionTrait, + S: ImmutableDeleteStore, +{ + for (space, value) in ordered_unique(blobs).iter() { + let tx = conn.begin().await?; + let key = (SpaceIdWrap(space.clone()), *value); + let row = if tx.get_database_backend() == DbBackend::Sqlite { + let updated = candidate::Entity::update_many() + .col_expr( + candidate::Column::CreatedAt, + Expr::col(candidate::Column::CreatedAt).into(), + ) + .filter(candidate::Column::Space.eq(key.0.clone())) + .filter(candidate::Column::Value.eq(key.1)) + .exec(&tx) + .await?; + if updated.rows_affected == 0 { + continue; + } + candidate::Entity::find_by_id(key).one(&tx).await? + } else { + candidate::Entity::find_by_id(key) + .lock_exclusive() + .one(&tx) + .await? + }; + if let Some(row) = row { + remove_if_unreferenced(&tx, store, row).await?; + } + tx.commit().await?; + } + Ok(()) +} + +pub async fn collect_candidates( + conn: &C, + store: &S, + max: u64, + min_age: Duration, +) -> Result> +where + C: ConnectionTrait + TransactionTrait, + S: ImmutableDeleteStore, +{ + Ok(collect_candidates_with_progress(conn, store, max, min_age) + .await? + .0) +} + +/// Collect at most `min(max, 256)` old candidates in one pass. +/// +/// A row lock excludes uploaders of the same `(space, hash)` until reference +/// checks, blob deletion and candidate removal have completed. Historical +/// `kv_write` references are as important as current KV references: a sync +/// client may still read an older value. +pub async fn collect_candidates_with_progress( + conn: &C, + store: &S, + max: u64, + min_age: Duration, +) -> Result<(usize, usize), CollectError> +where + C: ConnectionTrait + TransactionTrait, + S: ImmutableDeleteStore, +{ + let limit = max.min(MAX_BATCH_SIZE); + if limit == 0 { + return Ok((0, 0)); + } + let cutoff = OffsetDateTime::now_utc() + .checked_sub(min_age) + .unwrap_or(OffsetDateTime::UNIX_EPOCH); + let tx = conn.begin().await?; + let query = candidate::Entity::find() + .filter(candidate::Column::CreatedAt.lte(cutoff)) + .order_by_asc(candidate::Column::CreatedAt) + .order_by_asc(candidate::Column::Space) + .order_by_asc(candidate::Column::Value) + .limit(limit); + let query = if tx.get_database_backend() == DbBackend::Postgres { + query.lock_with_behavior(LockType::Update, LockBehavior::SkipLocked) + } else { + query + }; + let rows = query.all(&tx).await?; + let scanned = rows.len(); + let mut removed = 0; + for row in rows { + // PostgreSQL SELECT FOR UPDATE already owns this lock. MySQL + // obtains a fresh locking read; SQLite upgrades to a write lock. + if tx.get_database_backend() != DbBackend::Postgres { + let fresh = if tx.get_database_backend() == DbBackend::Sqlite { + let updated = candidate::Entity::update_many() + .col_expr( + candidate::Column::CreatedAt, + Expr::col(candidate::Column::CreatedAt).into(), + ) + .filter(candidate::Column::Space.eq(row.space.clone())) + .filter(candidate::Column::Value.eq(row.value)) + .exec(&tx) + .await?; + if updated.rows_affected == 0 { + continue; + } + candidate::Entity::find_by_id((row.space.clone(), row.value)) + .one(&tx) + .await? + } else { + candidate::Entity::find_by_id((row.space.clone(), row.value)) + .lock_exclusive() + .one(&tx) + .await? + }; + if !fresh.is_some_and(|candidate| candidate.created_at <= cutoff) { + continue; + } + } + + removed += usize::from(remove_if_unreferenced(&tx, store, row).await?); + } + tx.commit().await?; + Ok((removed, scanned)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hash::hash; + use crate::models::kv_orphan_candidate; + use crate::storage::{ + memory::MemoryStore, HashBuffer, ImmutableReadStore, ImmutableWriteStore, StoreSize, + }; + use futures::io::AsyncWriteExt; + use sea_orm::{ + sea_query::Expr, ConnectOptions, ConnectionTrait, Database, DatabaseConnection, DbBackend, + EntityTrait, QueryFilter, Statement, TransactionTrait, + }; + use sea_orm_migration::MigratorTrait; + use tinycloud_auth::{resolver::DID_METHODS, ssi::jwk::JWK}; + + async fn database() -> DatabaseConnection { + let db = Database::connect(ConnectOptions::new("sqlite::memory:".to_owned())) + .await + .unwrap(); + crate::migrations::Migrator::up(&db, None).await.unwrap(); + db.execute(Statement::from_string( + DbBackend::Sqlite, + "PRAGMA foreign_keys = OFF", + )) + .await + .unwrap(); + db + } + + fn space() -> SpaceId { + let jwk = JWK::generate_ed25519().unwrap(); + let did = DID_METHODS.generate(&jwk, "key").unwrap(); + SpaceId::new(did, "files".parse().unwrap()) + } + + async fn upload(store: &MemoryStore, space: &SpaceId, bytes: &[u8]) -> Hash { + let mut staged = HashBuffer::new(Vec::new()); + staged.write_all(bytes).await.unwrap(); + store.persist(space, staged).await.unwrap() + } + + async fn age(db: &DatabaseConnection, space: &SpaceId, value: Hash) { + kv_orphan_candidate::Entity::update_many() + .col_expr( + kv_orphan_candidate::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - time::Duration::hours(2)), + ) + .filter(kv_orphan_candidate::Column::Space.eq(crate::types::SpaceIdWrap(space.clone()))) + .filter(kv_orphan_candidate::Column::Value.eq(value)) + .exec(db) + .await + .unwrap(); + } + + async fn reference(db: &DatabaseConnection, table: &str, space: &SpaceId, value: Hash) { + let sql = if table == "kv_write" { + "INSERT INTO kv_write (space, key, invocation, seq, epoch, epoch_seq, value, metadata) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}')" + } else { + "INSERT INTO current_kv (space, key, invocation, seq, epoch, epoch_seq, value, metadata, deleted) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}', 0)" + }; + db.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + sql, + [ + space.to_string().into(), + Vec::::from(hash(table.as_bytes())).into(), + Vec::::from(hash(b"epoch")).into(), + Vec::::from(value).into(), + ], + )) + .await + .unwrap(); + } + + #[tokio::test] + async fn historical_and_current_references_survive_collection() { + let db = database().await; + let store = MemoryStore::default(); + let history_space = space(); + let current_space = space(); + let historical = upload(&store, &history_space, b"historical").await; + let current = upload(&store, ¤t_space, b"current").await; + let candidates = [ + (history_space.clone(), historical), + (current_space.clone(), current), + ]; + register_candidates(&db, &candidates).await.unwrap(); + for (space, hash) in &candidates { + age(&db, space, *hash).await; + } + reference(&db, "kv_write", &history_space, historical).await; + reference(&db, "current_kv", ¤t_space, current).await; + + assert_eq!( + collect_candidates(&db, &store, 10, time::Duration::hours(1)) + .await + .unwrap(), + 0 + ); + for (space, hash) in &candidates { + assert!(store.contains(space, hash).await.unwrap()); + assert!(kv_orphan_candidate::Entity::find_by_id(( + crate::types::SpaceIdWrap(space.clone()), + *hash + )) + .one(&db) + .await + .unwrap() + .is_none()); + } + } + + #[tokio::test] + async fn crashed_upload_is_refunded_after_five_minute_grace() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = hash(b"failed-precondition"); + register_candidates(&db, &[(space.clone(), value)]) + .await + .unwrap(); + assert_eq!(upload(&store, &space, b"failed-precondition").await, value); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(b"failed-precondition".len() as u64) + ); + let tx = db.begin().await.unwrap(); + lock_candidates(&tx, &[(space.clone(), value)]) + .await + .unwrap(); + tx.rollback().await.unwrap(); + age(&db, &space, value).await; + + assert_eq!( + collect_candidates(&db, &store, 10, time::Duration::minutes(5)) + .await + .unwrap(), + 1 + ); + assert!(!store.contains(&space, &value).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert!( + kv_orphan_candidate::Entity::find_by_id((crate::types::SpaceIdWrap(space), value)) + .one(&db) + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn grace_and_pass_limit_do_not_scan_or_delete_newer_rows() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let mut old = Vec::new(); + for bytes in [b"old-1".as_slice(), b"old-2", b"old-3"] { + let value = upload(&store, &space, bytes).await; + register_candidates(&db, &[(space.clone(), value)]) + .await + .unwrap(); + age(&db, &space, value).await; + old.push(value); + } + let fresh = upload(&store, &space, b"fresh").await; + register_candidates(&db, &[(space.clone(), fresh)]) + .await + .unwrap(); + assert_eq!( + collect_candidates(&db, &store, 2, time::Duration::hours(1)) + .await + .unwrap(), + 2 + ); + let mut surviving = 0; + for value in &old { + surviving += usize::from(store.contains(&space, value).await.unwrap()); + } + assert_eq!(surviving, 1); + assert!(store.contains(&space, &fresh).await.unwrap()); + assert_eq!( + collect_candidates(&db, &store, 2, time::Duration::hours(1)) + .await + .unwrap(), + 1 + ); + assert_eq!( + collect_candidates(&db, &store, 2, time::Duration::hours(1)) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &fresh).await.unwrap()); + } + + #[tokio::test] + async fn registering_existing_hash_resets_grace_period() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"same-hash").await; + register_candidates(&db, &[(space.clone(), value)]) + .await + .unwrap(); + age(&db, &space, value).await; + register_candidates(&db, &[(space.clone(), value)]) + .await + .unwrap(); + + assert_eq!( + collect_candidates(&db, &store, 1, time::Duration::hours(1)) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &value).await.unwrap()); + } + + #[tokio::test] + async fn uploader_recreates_candidate_deleted_by_earlier_same_hash_commit() { + let db = database().await; + let space = space(); + let value = hash(b"same-hash"); + let blobs = [(space.clone(), value)]; + register_candidates(&db, &blobs).await.unwrap(); + // The next uploader has already pre-registered this same hash. + let earlier = db.begin().await.unwrap(); + lock_candidates(&earlier, &blobs).await.unwrap(); + finish_candidates(&earlier, &blobs).await.unwrap(); + earlier.commit().await.unwrap(); + assert!( + kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space.clone()), value)) + .one(&db) + .await + .unwrap() + .is_none() + ); + let later = db.begin().await.unwrap(); + lock_candidates(&later, &blobs).await.unwrap(); + assert!( + kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space), value)) + .one(&later) + .await + .unwrap() + .is_some(), + "the second upload must have a durable hash fence after the first commit" + ); + later.commit().await.unwrap(); + } + + #[tokio::test] + async fn postgres_upload_fence_skips_locked_candidate() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_upload_fence_skips_locked_candidate", + ) + .await + else { + return; + }; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"upload-in-progress").await; + register_candidates(&fixture.conn, &[(space.clone(), value)]) + .await + .unwrap(); + kv_orphan_candidate::Entity::update_many() + .col_expr( + kv_orphan_candidate::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - time::Duration::hours(2)), + ) + .filter(kv_orphan_candidate::Column::Space.eq(crate::types::SpaceIdWrap(space.clone()))) + .filter(kv_orphan_candidate::Column::Value.eq(value)) + .exec(&fixture.conn) + .await + .unwrap(); + let tx = fixture.conn.begin().await.unwrap(); + lock_candidates(&tx, &[(space.clone(), value)]) + .await + .unwrap(); + let collected = tokio::time::timeout( + std::time::Duration::from_secs(5), + collect_candidates(&fixture.conn, &store, 1, time::Duration::hours(1)), + ) + .await + .expect("collector should skip the locked candidate") + .unwrap(); + assert_eq!(collected, 0); + assert!(store.contains(&space, &value).await.unwrap()); + tx.rollback().await.unwrap(); + assert_eq!( + collect_candidates(&fixture.conn, &store, 1, time::Duration::hours(1)) + .await + .unwrap(), + 1 + ); + } + + #[tokio::test] + async fn successful_commit_removes_candidate_without_collecting_blob() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = hash(b"committed"); + register_candidates(&db, &[(space.clone(), value)]) + .await + .unwrap(); + let tx = db.begin().await.unwrap(); + lock_candidates(&tx, &[(space.clone(), value)]) + .await + .unwrap(); + assert_eq!(upload(&store, &space, b"committed").await, value); + finish_candidates(&tx, &[(space.clone(), value)]) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + collect_candidates(&db, &store, 10, time::Duration::ZERO) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &value).await.unwrap()); + } +} diff --git a/tinycloud-core/src/kv_sync.rs b/tinycloud-core/src/kv_sync.rs index 891d308d..0a49c542 100644 --- a/tinycloud-core/src/kv_sync.rs +++ b/tinycloud-core/src/kv_sync.rs @@ -1650,6 +1650,250 @@ pub(crate) mod tests { ); } + #[tokio::test] + async fn unauthorized_put_never_uploads_a_blob() { + use crate::storage::ImmutableReadStore; + let store = MemoryStore::default(); + let db = SpaceDatabase::new( + Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(), + store.clone(), + StaticSecret::new([0u8; 32].to_vec()).unwrap(), + ) + .await + .unwrap(); + let signer = Owner::generate(); + let other = Owner::generate(); + host(&db, &other).await; + let op = Op::Put("notes/forbidden", b"unauthorized"); + let result = invoke_in( + &db, + &signer, + &[(&other.space, &op)], + KvInvokeOptions::default(), + ) + .await; + assert!( + result.is_err(), + "the signer cannot write another owner's space" + ); + assert!( + !store + .contains(&other.space, &crate::hash::hash(b"unauthorized")) + .await + .unwrap(), + "authorization failure must not create a blob" + ); + } + + #[tokio::test] + async fn rejected_put_refunds_quota_without_deleting_committed_hash() { + use crate::storage::{ImmutableReadStore, StoreSize}; + let store = MemoryStore::default(); + let db = SpaceDatabase::new( + Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(), + store.clone(), + StaticSecret::new([0u8; 32].to_vec()).unwrap(), + ) + .await + .unwrap(); + let owner = Owner::generate(); + host(&db, &owner).await; + let committed = run(&db, &owner, &[Op::Put("notes/a", b"kept")]).await[0]; + let mut options = KvInvokeOptions::default(); + options.preconditions.insert( + (owner.space.clone(), "notes/a".parse().unwrap()), + KvPrecondition::DoesNotExist, + ); + let rejected = invoke(&db, &owner, &[Op::Put("notes/a", b"rejected")], options) + .await + .unwrap_err(); + assert!(rejected.contains("precondition"), "{rejected}"); + let orphan = crate::hash::hash(b"rejected"); + assert!( + !store.contains(&owner.space, &orphan).await.unwrap(), + "a known precondition failure must remove its unreferenced upload" + ); + assert!(store.contains(&owner.space, &committed).await.unwrap()); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(4)); + let duplicate = invoke(&db, &owner, &[Op::Put("notes/a", b"kept")], { + let mut options = KvInvokeOptions::default(); + options.preconditions.insert( + (owner.space.clone(), "notes/a".parse().unwrap()), + KvPrecondition::DoesNotExist, + ); + options + }) + .await + .unwrap_err(); + assert!(duplicate.contains("precondition"), "{duplicate}"); + assert!(store.contains(&owner.space, &committed).await.unwrap()); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(4)); + } + + #[tokio::test] + async fn failed_database_write_refunds_uploaded_blob_immediately() { + use crate::storage::{ImmutableReadStore, StoreSize}; + use sea_orm::{ConnectionTrait, DbBackend, Statement}; + let store = MemoryStore::default(); + let db = SpaceDatabase::new( + Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(), + store.clone(), + StaticSecret::new([0u8; 32].to_vec()).unwrap(), + ) + .await + .unwrap(); + let owner = Owner::generate(); + host(&db, &owner).await; + db.connection() + .execute(Statement::from_string( + DbBackend::Sqlite, + "CREATE TRIGGER fail_kv_write BEFORE INSERT ON kv_write BEGIN SELECT RAISE(FAIL, 'blocked'); END", + )) + .await + .unwrap(); + let error = invoke( + &db, + &owner, + &[Op::Put("notes/failure", b"uploaded")], + KvInvokeOptions::default(), + ) + .await + .unwrap_err(); + assert!(error.contains("blocked"), "{error}"); + assert!(!store + .contains(&owner.space, &crate::hash::hash(b"uploaded")) + .await + .unwrap()); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); + } + + /// Simulate an object store that accepted the PUT but lost its success + /// response. The caller must still delete and refund the possible object. + #[derive(Clone, Default)] + struct AmbiguousUploadStore(MemoryStore); + + #[async_trait::async_trait] + impl crate::storage::StorageSetup for AmbiguousUploadStore { + type Error = std::io::Error; + async fn create(&self, space: &SpaceId) -> Result<(), Self::Error> { + self.0.create(space).await + } + } + + #[async_trait::async_trait] + impl crate::storage::ImmutableReadStore for AmbiguousUploadStore { + type Error = std::io::Error; + type Readable = futures::io::Cursor>; + async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { + self.0.contains(space, id).await + } + async fn read( + &self, + space: &SpaceId, + id: &Hash, + ) -> Result>, Self::Error> { + self.0.read(space, id).await + } + async fn read_range( + &self, + space: &SpaceId, + id: &Hash, + range: crate::storage::ByteRangeSpec, + ) -> Result>, Self::Error> { + self.0.read_range(space, id, range).await + } + } + + #[async_trait::async_trait] + impl crate::storage::ImmutableWriteStore for AmbiguousUploadStore { + type Error = std::io::Error; + async fn persist( + &self, + space: &SpaceId, + staged: HashBuffer>, + ) -> Result { + self.0.persist(space, staged).await?; + Err(std::io::Error::other("lost PUT response")) + } + } + + #[async_trait::async_trait] + impl crate::storage::ImmutableDeleteStore for AmbiguousUploadStore { + type Error = std::io::Error; + async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error> { + self.0.remove(space, id).await + } + } + + #[tokio::test] + async fn failed_upload_refunds_physical_bytes_immediately() { + use crate::storage::{ImmutableReadStore, StoreSize}; + let store = AmbiguousUploadStore::default(); + let db = SpaceDatabase::new( + Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(), + store.clone(), + StaticSecret::new([0u8; 32].to_vec()).unwrap(), + ) + .await + .unwrap(); + let owner = Owner::generate(); + space::ActiveModel { + id: Set(SpaceIdWrap(owner.space.clone())), + } + .insert(db.connection()) + .await + .unwrap(); + let op = Op::Put("notes/failure", b"uploaded"); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"uploaded").await.unwrap(); + let mut inputs = std::collections::HashMap::new(); + inputs.insert( + ( + owner.space.clone(), + "notes/failure".parse::().unwrap(), + ), + (Metadata(Default::default()), stage), + ); + let error = match db + .invoke::(owner.invocation_in(&[(&owner.space, &op)]), inputs) + .await + { + Ok(_) => panic!("the upload response was lost"), + Err(error) => error.to_string(), + }; + assert!(error.contains("lost PUT response"), "{error}"); + assert!(!store + .contains(&owner.space, &crate::hash::hash(b"uploaded")) + .await + .unwrap()); + assert_eq!(store.0.total_size(&owner.space).await.unwrap(), Some(0)); + assert_eq!( + crate::kv_orphans::collect_candidates( + db.connection(), + &store, + 1, + time::Duration::ZERO, + ) + .await + .unwrap(), + 0, + "the immediate failure path consumed the candidate" + ); + assert_eq!( + store.0.total_size(&owner.space).await.unwrap(), + Some(0), + "a later GC pass must not refund the same object twice" + ); + } + // ── PostgreSQL (run in CI's PG16 job; `TINYCLOUD_TEST_POSTGRES_URL`) ── /// Drops its schema when it goes out of scope, including while a failed @@ -1697,6 +1941,7 @@ pub(crate) mod tests { /// ("sorry, too many clients already" at `CREATE SCHEMA`). struct PostgresFixture { dbs: Vec, + store: MemoryStore, _schema: SchemaGuard, // Last, so the schema is dropped before the next fixture starts. _serial: tokio::sync::MutexGuard<'static, ()>, @@ -1728,6 +1973,7 @@ pub(crate) mod tests { schema: schema.clone(), }; let mut dbs = Vec::with_capacity(instances); + let store = MemoryStore::default(); for _ in 0..instances { let mut options = ConnectOptions::new(url.clone()); options @@ -1739,7 +1985,7 @@ pub(crate) mod tests { Database::connect(options) .await .expect("connect to isolated TC-732 schema"), - MemoryStore::default(), + store.clone(), StaticSecret::new([0u8; 32].to_vec()).unwrap(), ) .await @@ -1748,12 +1994,53 @@ pub(crate) mod tests { } Some(Self { dbs, + store, _schema: guard, _serial: serial, }) } } + /// PG's per-space lock may be held by another process, but independent + /// content-addressed uploads must still finish before acquiring it. + #[tokio::test] + async fn postgres_blob_persists_before_space_sequence_lock() { + use crate::storage::ImmutableReadStore; + let Some(fixture) = + PostgresFixture::new("postgres_blob_persists_before_space_sequence_lock", 2).await + else { + return; + }; + let owner = std::sync::Arc::new(Owner::generate()); + host(&fixture.dbs[0], &owner).await; + let holder = fixture.dbs[0].connection().begin().await.unwrap(); + lock_space_sequences(&holder, [&owner.space]).await.unwrap(); + let db = fixture.dbs[1].clone(); + let writer_owner = owner.clone(); + let writer = tokio::spawn(async move { + run( + &db, + &writer_owner, + &[Op::Put("notes/held", b"persist-before-lock")], + ) + .await + }); + let hash = crate::hash::hash(b"persist-before-lock"); + let visible = tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if fixture.store.contains(&owner.space, &hash).await.unwrap() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .is_ok(); + holder.commit().await.unwrap(); + writer.await.unwrap(); + assert!(visible, "blob upload must not wait for the sequence lock"); + } + /// `(seq, epoch bytes, epoch_seq)`: the order the feed promises. fn order_key(pos: &KvSyncAnchor) -> (i64, Vec, i64) { (pos.seq, Vec::::from(pos.epoch), pos.epoch_seq) @@ -2134,6 +2421,7 @@ pub(crate) mod tests { #[tokio::test] async fn postgres_conditional_put_is_atomic_under_concurrency() { use crate::models::kv_write; + use crate::storage::StoreSize; const INSTANCES: usize = 4; const CONTENDERS: usize = 32; let Some(fixture) = PostgresFixture::new( @@ -2210,6 +2498,7 @@ pub(crate) mod tests { who(&created) ); let mut committed = vec![created[0].1]; + let mut committed_bytes = format!("create-{}", created[0].0).len() as u64; // Before the first replace no etag has been replaced yet: present // one that was never written. let mut stale = crate::hash::hash(b"never written"); @@ -2232,6 +2521,7 @@ pub(crate) mod tests { ); stale = current; committed.push(replaced[0].1); + committed_bytes += format!("replace{round}-{}", replaced[0].0).len() as u64; } run(&fixture.dbs[1], &owner, &[Op::Del("notes/x")]).await; let recreated = race("recreate".into(), create()).await; @@ -2242,6 +2532,7 @@ pub(crate) mod tests { who(&recreated) ); committed.push(recreated[0].1); + committed_bytes += format!("recreate-{}", recreated[0].0).len() as u64; let over_live = race("create-over-live".into(), create()).await; assert!( over_live.is_empty(), @@ -2271,6 +2562,11 @@ pub(crate) mod tests { writes.iter().map(|write| write.value).collect::>(), committed ); + assert_eq!( + fixture.store.total_size(&owner.space).await.unwrap(), + Some(committed_bytes), + "failed contenders must refund their physical bytes without touching committed blobs" + ); assert_eq!( assert_linear_history(conn, &owner.space, "conditional").await, committed.len() + 1 diff --git a/tinycloud-core/src/lib.rs b/tinycloud-core/src/lib.rs index 3cacd735..eef711b7 100644 --- a/tinycloud-core/src/lib.rs +++ b/tinycloud-core/src/lib.rs @@ -13,6 +13,7 @@ pub mod encryption_network; pub mod events; pub mod hash; pub mod keys; +pub mod kv_orphans; pub mod kv_sync; pub mod manifest; pub mod meeting_legacy_guard; diff --git a/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs b/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs new file mode 100644 index 00000000..ef0c557f --- /dev/null +++ b/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs @@ -0,0 +1,35 @@ +use sea_orm::Schema; +use sea_orm_migration::prelude::*; + +use crate::models::kv_orphan_candidate; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let schema = Schema::new(manager.get_database_backend()); + manager + .create_table(schema.create_table_from_entity(kv_orphan_candidate::Entity)) + .await?; + manager + .create_index( + Index::create() + .name("idx_kv_orphan_candidate_age") + .table(kv_orphan_candidate::Entity) + .col(kv_orphan_candidate::Column::CreatedAt) + .col(kv_orphan_candidate::Column::Space) + .col(kv_orphan_candidate::Column::Value) + .to_owned(), + ) + .await?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(kv_orphan_candidate::Entity).to_owned()) + .await + } +} diff --git a/tinycloud-core/src/migrations/mod.rs b/tinycloud-core/src/migrations/mod.rs index ee7f0d2f..0aac3c2c 100644 --- a/tinycloud-core/src/migrations/mod.rs +++ b/tinycloud-core/src/migrations/mod.rs @@ -25,6 +25,7 @@ pub mod m20261005_000000_deactivate_hook_subscriptions; pub mod m20261007_000000_database_alias; pub mod m20261007_010000_database_identity_fence; pub mod m20261015_000000_delegation_cursor_index; +pub mod m20261015_020000_kv_orphan_candidates; pub struct Migrator; @@ -58,6 +59,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_000000_database_alias::Migration), Box::new(m20261007_010000_database_identity_fence::Migration), Box::new(m20261015_000000_delegation_cursor_index::Migration), + Box::new(m20261015_020000_kv_orphan_candidates::Migration), ] } } diff --git a/tinycloud-core/src/models/kv_orphan_candidate.rs b/tinycloud-core/src/models/kv_orphan_candidate.rs new file mode 100644 index 00000000..efbfdc13 --- /dev/null +++ b/tinycloud-core/src/models/kv_orphan_candidate.rs @@ -0,0 +1,19 @@ +use crate::hash::Hash; +use crate::types::SpaceIdWrap; +use sea_orm::entity::prelude::*; +use time::OffsetDateTime; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "kv_orphan_candidate")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub space: SpaceIdWrap, + #[sea_orm(primary_key, auto_increment = false)] + pub value: Hash, + pub created_at: OffsetDateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/tinycloud-core/src/models/mod.rs b/tinycloud-core/src/models/mod.rs index a8a0afca..9cdd3e48 100644 --- a/tinycloud-core/src/models/mod.rs +++ b/tinycloud-core/src/models/mod.rs @@ -18,6 +18,7 @@ pub mod hook_subscription; pub mod invocation; pub mod invocation_replay; pub mod kv_delete; +pub mod kv_orphan_candidate; pub mod kv_write; pub mod meeting_legacy_write_guard; pub mod owner_share_policy; diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index f2c49408..3cc978fd 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -622,14 +622,12 @@ pub async fn app_with_control( encryption_backend, ); - let tinycloud = TinyCloud::new( - database_connection, - tinycloud_config.storage.blocks.open().await?, - key_setup.setup(()).await?, - ) - .await? - .with_encryption(Some(webhook_encryption.clone())) - .with_sql_sizes(sql_sizes.clone()); + let block_store = tinycloud_config.storage.blocks.open().await?; + let orphan_store = block_store.clone(); + let tinycloud = TinyCloud::new(database_connection, block_store, key_setup.setup(()).await?) + .await? + .with_encryption(Some(webhook_encryption.clone())) + .with_sql_sizes(sql_sizes.clone()); let encryption_service = encryption_service.with_sqlite_writer_lock(tinycloud.sqlite_writer_lock()); let database_artifact_repository: Arc = @@ -719,6 +717,8 @@ pub async fn app_with_control( )?; spawn_webhook_dispatcher(webhook_dispatcher); + let orphan_conn = seed_conn.clone(); + // TC-326: capture handles for the telemetry DB sampler before `tinycloud` // is moved into Rocket-managed state below. let telemetry_enabled = tinycloud_config.telemetry.enabled; @@ -769,6 +769,36 @@ pub async fn app_with_control( }) }, )) + .attach(AdHoc::on_liftoff("kv-orphan-cleanup", move |rocket| { + let conn = orphan_conn.clone(); + let store = orphan_store.clone(); + let shutdown = rocket.shutdown(); + Box::pin(async move { + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(60)); + tokio::pin!(shutdown); + loop { + tokio::select! { + _ = interval.tick() => { + loop { + match tinycloud_core::kv_orphans::collect_candidates_with_progress( + &conn, &store, 256, time::Duration::minutes(5), + ).await { + Ok((_, 256)) => continue, + Ok(_) => break, + Err(error) => { + ::tracing::warn!(%error, "KV orphan collection failed"); + break; + } + } + } + } + _ = &mut shutdown => break, + } + } + }); + }) + })) .manage(tinycloud) .manage(node_public_keys) .manage(sql_service); diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index 1896dbb5..b05c4940 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -9,9 +9,10 @@ use futures::{ use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, + collections::{hash_map::Entry, HashMap}, io::{Error as IoError, ErrorKind, SeekFrom}, path::{Path, PathBuf}, + sync::Arc, time::Instant, }; use tempfile::{NamedTempFile, PathPersistError}; @@ -20,6 +21,7 @@ use tinycloud_core::{hash::Hash, storage::*}; use tokio::{ fs::{create_dir_all, metadata, remove_file, File}, io::{AsyncReadExt, AsyncSeekExt}, + sync::Mutex, }; use tokio_stream::wrappers::ReadDirStream; @@ -29,13 +31,20 @@ use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; pub struct FileSystemStore { path: PathBuf, sizes: SpaceSizes, + // A partial filesystem write can fail after creating a file. Reserve + // before writing and refund the full reservation when deleting it. + pending: Arc>>, } impl FileSystemStore { async fn new(path: PathBuf) -> Result { // get the size of the directory let sizes = store_sizes(&path).await?.into(); - Ok(Self { path, sizes }) + Ok(Self { + path, + sizes, + pending: Arc::new(Mutex::new(HashMap::new())), + }) } fn get_path(&self, space: &SpaceId, mh: &Hash) -> PathBuf { @@ -51,6 +60,61 @@ impl FileSystemStore { async fn decrement_size(&self, space: &SpaceId, size: u64) { self.sizes.decrement_size(space, size).await; } + + async fn reserve(&self, space: &SpaceId, hash: Hash, size: u64) { + let mut pending = self.pending.lock().await; + if let Entry::Vacant(slot) = pending.entry((space.clone(), hash)) { + slot.insert(size); + self.increment_size(space, size).await; + } + } + + async fn settle(&self, space: &SpaceId, hash: Hash) { + self.pending.lock().await.remove(&(space.clone(), hash)); + } + + async fn refund_missing_pending(&self, space: &SpaceId, hash: Hash) { + if let Some(size) = self.pending.lock().await.remove(&(space.clone(), hash)) { + self.decrement_size(space, size).await; + } + } + + async fn persist_temp( + &self, + space: &SpaceId, + hash: Hash, + file: TempFileStage, + ) -> Result<(), FileSystemStoreError> { + if self.contains(space, &hash).await? { + self.settle(space, hash).await; + return Ok(()); + } + let size = file.size().await?; + self.reserve(space, hash, size).await; + let (_, path) = file.into_inner(); + path.persist(self.get_path(space, &hash))?; + self.settle(space, hash).await; + Ok(()) + } + + async fn persist_memory( + &self, + space: &SpaceId, + hash: Hash, + bytes: Vec, + ) -> Result<(), FileSystemStoreError> { + if self.contains(space, &hash).await? { + self.settle(space, hash).await; + return Ok(()); + } + self.reserve(space, hash, bytes.len() as u64).await; + let file = File::create(self.get_path(space, &hash)).await?; + let mut writer = futures::io::BufWriter::new(file.compat()); + writer.write_all(&bytes).await?; + writer.flush().await?; + self.settle(space, hash).await; + Ok(()) + } } #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] @@ -277,12 +341,7 @@ impl ImmutableWriteStore for FileSystemStore { let result = async move { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - if !self.contains(space, &hash).await? { - let size = f.size().await?; - let (_, path) = f.into_inner(); - path.persist(self.get_path(space, &hash))?; - self.increment_size(space, size).await; - } + self.persist_temp(space, hash, f).await?; Ok(hash) } .await; @@ -307,14 +366,7 @@ impl ImmutableWriteStore for FileSystemStore { let result = async move { let (mut h, v) = staged.into_inner(); let hash = h.finalize(); - if !self.contains(space, &hash).await? { - let file = File::create(self.get_path(space, &hash)).await?; - let size = v.len() as u64; - let mut writer = futures::io::BufWriter::new(file.compat()); - writer.write_all(&v).await?; - writer.flush().await?; - self.increment_size(space, size).await; - } + self.persist_memory(space, hash, v).await?; Ok(hash) } .await; @@ -342,24 +394,10 @@ impl ImmutableWriteStore { - let size = t_file.size().await?; - let (_, path) = t_file.into_inner(); - path.persist(self.get_path(space, &hash))?; - self.increment_size(space, size).await; - } - AsyncEither::Right(v) => { - let file = File::create(self.get_path(space, &hash)).await?; - let size = v.len() as u64; - let mut writer = futures::io::BufWriter::new(file.compat()); - writer.write_all(&v).await?; - writer.flush().await?; - self.increment_size(space, size).await; - } - } - }; + match f { + AsyncEither::Left(file) => self.persist_temp(space, hash, file).await?, + AsyncEither::Right(bytes) => self.persist_memory(space, hash, bytes).await?, + } Ok(hash) } .await; @@ -379,15 +417,27 @@ impl ImmutableDeleteStore for FileSystemStore { let path = self.get_path(space, id); let size = match metadata(&path).await { Ok(m) => m.len(), - Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) if e.kind() == ErrorKind::NotFound => { + self.refund_missing_pending(space, *id).await; + return Ok(None); + } Err(e) => return Err(e.into()), }; match remove_file(path).await { Ok(()) => { - self.decrement_size(space, size).await; + let charged = self + .pending + .lock() + .await + .remove(&(space.clone(), *id)) + .unwrap_or(size); + self.decrement_size(space, charged).await; Ok(Some(())) } - Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), + Err(e) if e.kind() == ErrorKind::NotFound => { + self.refund_missing_pending(space, *id).await; + Ok(None) + } Err(e) => Err(e.into()), } } @@ -404,7 +454,30 @@ impl StorageConfig for TempFileSystemStage { #[cfg(test)] mod test { use super::*; - use futures::io::AsyncReadExt; + use futures::io::{AsyncReadExt, AsyncWriteExt}; + + #[tokio::test] + async fn failed_file_creation_refunds_reserved_quota() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:failed".parse().unwrap(); + store.create(&space).await.unwrap(); + tokio::fs::remove_dir(store.path.join(space.suffix()).join(space.name().as_str())) + .await + .unwrap(); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"failed-write").await.unwrap(); + let hash = stage.hash(); + assert!( + ImmutableWriteStore::::persist(&store, &space, stage) + .await + .is_err() + ); + assert_eq!(store.total_size(&space).await.unwrap(), Some(12)); + assert_eq!(store.remove(&space, &hash).await.unwrap(), None); + assert!(!store.contains(&space, &hash).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } #[tokio::test] async fn test_file_system_store() { diff --git a/tinycloud-node-server/src/storage/s3.rs b/tinycloud-node-server/src/storage/s3.rs index 8c88c07d..9322743c 100644 --- a/tinycloud-node-server/src/storage/s3.rs +++ b/tinycloud-node-server/src/storage/s3.rs @@ -1,8 +1,5 @@ use aws_sdk_s3::{ - error::{ - GetObjectAttributesError, GetObjectAttributesErrorKind, GetObjectError, GetObjectErrorKind, - HeadObjectError, HeadObjectErrorKind, - }, + error::{GetObjectError, GetObjectErrorKind, HeadObjectError, HeadObjectErrorKind}, types::{ByteStream, SdkError}, Client, // Config, Error as S3Error, @@ -16,11 +13,19 @@ use futures::{ use rocket::{async_trait, http::hyper::Uri}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; -use std::{collections::HashMap, io::Error as IoError, ops::AddAssign, time::Instant}; +use std::{ + collections::{hash_map::Entry, HashMap}, + future::Future, + io::Error as IoError, + ops::AddAssign, + sync::Arc, + time::Instant, +}; use tinycloud_auth::resource::SpaceId; use tinycloud_core::{hash::Hash, storage::*}; use super::{file_system, size::SpaceSizes}; +use tokio::sync::Mutex; async fn aws_config() -> SdkConfig { aws_config::from_env().load().await @@ -31,6 +36,10 @@ pub struct S3BlockStore { pub client: Client, pub bucket: String, sizes: SpaceSizes, + // A PUT can succeed on S3 even if its response fails. Reserve its size + // before sending; this shared map refunds a missing object on known + // failure without subtracting another writer's committed bytes. + pending: Arc>>, } #[serde_as] @@ -102,6 +111,7 @@ impl S3BlockStore { client, bucket: config.bucket.clone(), sizes, + pending: Arc::new(Mutex::new(HashMap::new())), }) } @@ -119,6 +129,47 @@ impl S3BlockStore { async fn decrement_size(&self, space: &SpaceId, size: u64) { self.sizes.decrement_size(space, size).await; } + + async fn put_if_absent( + &self, + space: &SpaceId, + hash: Hash, + body: F, + ) -> Result<(), S3StoreError> + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let key = (space.clone(), hash); + if self.contains(space, &hash).await? { + // A retry of an ambiguous PUT can find its original object. + self.pending.lock().await.remove(&key); + return Ok(()); + } + let (size, body) = body().await?; + let mut pending = self.pending.lock().await; + if let Entry::Vacant(slot) = pending.entry(key.clone()) { + slot.insert(size); + self.increment_size(space, size).await; + } + drop(pending); + self.client + .put_object() + .bucket(&self.bucket) + .key(self.key(space, &hash)) + .body(body) + .send() + .await + .map_err(S3Error::from)?; + self.pending.lock().await.remove(&key); + Ok(()) + } + + async fn refund_missing_pending(&self, space: &SpaceId, id: &Hash) { + if let Some(size) = self.pending.lock().await.remove(&(space.clone(), *id)) { + self.decrement_size(space, size).await; + } + } } pub fn convert(e: ByteStreamError) -> IoError { @@ -312,18 +363,10 @@ impl ImmutableWriteStore for S3BlockStore { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - if !self.contains(space, &hash).await? { - let size = f.len() as u64; - self.client - .put_object() - .bucket(&self.bucket) - .key(self.key(space, &hash)) - .body(ByteStream::from(f)) - .send() - .await - .map_err(S3Error::from)?; - self.increment_size(space, size).await; - } + self.put_if_absent(space, hash, || async move { + Ok((f.len() as u64, ByteStream::from(f))) + }) + .await?; Ok(hash) } .await; @@ -349,20 +392,12 @@ impl ImmutableWriteStore for S3BlockStore { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - if !self.contains(space, &hash).await? { + self.put_if_absent(space, hash, || async move { let size = f.size().await?; let (_file, path) = f.into_inner(); - - self.client - .put_object() - .bucket(&self.bucket) - .key(self.key(space, &hash)) - .body(ByteStream::from_path(&path).await?) - .send() - .await - .map_err(S3Error::from)?; - self.increment_size(space, size).await; - } + Ok((size, ByteStream::from_path(&path).await?)) + }) + .await?; Ok(hash) } .await; @@ -390,35 +425,22 @@ impl ImmutableWriteStore { + match f { + AsyncEither::Left(t_file) => { + self.put_if_absent(space, hash, || async move { let size = t_file.size().await?; let (_file, path) = t_file.into_inner(); - self.client - .put_object() - .bucket(&self.bucket) - .key(self.key(space, &hash)) - .body(ByteStream::from_path(&path).await?) - .send() - .await - .map_err(S3Error::from)?; - self.increment_size(space, size).await; - } - AsyncEither::Right(b) => { - let size = b.len() as u64; - self.client - .put_object() - .bucket(&self.bucket) - .key(self.key(space, &hash)) - .body(ByteStream::from(b)) - .send() - .await - .map_err(S3Error::from)?; - self.increment_size(space, size).await; - } + Ok((size, ByteStream::from_path(&path).await?)) + }) + .await?; } - }; + AsyncEither::Right(b) => { + self.put_if_absent(space, hash, || async move { + Ok((b.len() as u64, ByteStream::from(b))) + }) + .await?; + } + } Ok(hash) } .await; @@ -437,23 +459,25 @@ impl ImmutableDeleteStore for S3BlockStore { async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error> { let size: u64 = match self .client - .get_object_attributes() + .head_object() .bucket(&self.bucket) .key(self.key(space, id)) .send() .await { - Ok(o) if !o.delete_marker() => o.object_size().try_into()?, - Ok(_) => return Ok(None), + Ok(object) => object.content_length().try_into()?, Err(SdkError::ServiceError { err: - GetObjectAttributesError { - kind: GetObjectAttributesErrorKind::NoSuchKey(_), + HeadObjectError { + kind: HeadObjectErrorKind::NotFound(_), .. }, .. - }) => return Ok(None), - Err(e) => return Err(S3Error::from(e).into()), + }) => { + self.refund_missing_pending(space, id).await; + return Ok(None); + } + Err(error) => return Err(S3Error::from(error).into()), }; match self .client @@ -464,7 +488,13 @@ impl ImmutableDeleteStore for S3BlockStore { .await { Ok(_) => { - self.decrement_size(space, size).await; + let charged = self + .pending + .lock() + .await + .remove(&(space.clone(), *id)) + .unwrap_or(size); + self.decrement_size(space, charged).await; Ok(Some(())) } // TODO does this distinguish between object missing and object present? diff --git a/tinycloud-node-server/tests/postgres_kv_write_throughput.rs b/tinycloud-node-server/tests/postgres_kv_write_throughput.rs index 1fe561d4..24ff22da 100644 --- a/tinycloud-node-server/tests/postgres_kv_write_throughput.rs +++ b/tinycloud-node-server/tests/postgres_kv_write_throughput.rs @@ -49,7 +49,8 @@ use tinycloud_core::sea_orm::{ }; use tinycloud_core::storage::memory::{MemoryStaging, MemoryStore}; use tinycloud_core::storage::{ - HashBuffer, ImmutableReadStore, ImmutableWriteStore, StorageConfig, StorageSetup, + HashBuffer, ImmutableDeleteStore, ImmutableReadStore, ImmutableWriteStore, StorageConfig, + StorageSetup, StoreSize, }; use tinycloud_core::types::{Metadata, SpaceIdWrap}; use tinycloud_core::{SpaceDatabase, TxError, TxStoreError}; @@ -68,6 +69,7 @@ type Db = SpaceDatabase; trait BenchStore: ImmutableWriteStore + ImmutableReadStore + + ImmutableDeleteStore + StorageSetup + Clone + Send @@ -79,6 +81,7 @@ trait BenchStore: impl BenchStore for B where B: ImmutableWriteStore + ImmutableReadStore + + ImmutableDeleteStore + StorageSetup + Clone + Send @@ -182,6 +185,8 @@ struct CellReport { attempted: usize, ok: usize, wall: Duration, + started_at: time::OffsetDateTime, + ended_at: time::OffsetDateTime, p50: Duration, p95: Duration, errors: BTreeMap, @@ -322,6 +327,7 @@ async fn run_cell( batches.push(batch); } + let started_at = time::OffsetDateTime::now_utc(); let start = Instant::now(); let handles = batches .into_iter() @@ -361,6 +367,7 @@ async fn run_cell( } } let wall = start.elapsed(); + let ended_at = time::OffsetDateTime::now_utc(); latencies.sort_unstable(); let spaces = owners.iter().map(|o| o.space.clone()).collect::>(); @@ -372,6 +379,8 @@ async fn run_cell( attempted: latencies.len(), ok, wall, + started_at, + ended_at, p50: percentile(&latencies, 50), p95: percentile(&latencies, 95), errors, @@ -398,9 +407,9 @@ fn print_table(store: &str, reports: &[CellReport]) { println!(); println!( "| layout | concurrency | puts | ok | errors | ok ops/s | p50 ms | p95 ms | \ - event_order rows | dup (space,seq) groups | error kinds |" + event_order rows | dup (space,seq) groups | UTC start | UTC end | error kinds |" ); - println!("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|"); + println!("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|"); for r in reports { let error_count = r.attempted - r.ok; let kinds = if r.errors.is_empty() { @@ -413,7 +422,7 @@ fn print_table(store: &str, reports: &[CellReport]) { .join("; ") }; println!( - "| {} | {} | {} | {} | {} | {:.1} | {:.2} | {:.2} | {} | {} | {} |", + "| {} | {} | {} | {} | {} | {:.1} | {:.2} | {:.2} | {} | {} | {} | {} | {} |", r.layout.label(), r.concurrency, r.attempted, @@ -424,6 +433,8 @@ fn print_table(store: &str, reports: &[CellReport]) { ms(r.p95), r.event_rows, r.duplicate_seq_groups, + r.started_at, + r.ended_at, kinds, ); } @@ -515,6 +526,105 @@ async fn postgres_kv_write_throughput() { print_table(&store, &reports); } +/// Exercise the real S3 delete/accounting path for a rolled-back upload. +/// The regular benchmark intentionally has no failed writes. +#[tokio::test] +#[ignore = "manual PG16 + S3 integration test"] +async fn postgres_s3_orphan_cleanup_reclaims_space_usage() { + let database_url = std::env::var("TINYCLOUD_TEST_POSTGRES_URL").unwrap(); + let config: S3BlockConfig = serde_json::from_value(serde_json::json!({ + "bucket": std::env::var("TC732_BENCH_S3_BUCKET").unwrap(), + "endpoint": std::env::var("TC732_BENCH_S3_ENDPOINT").unwrap(), + })) + .unwrap(); + let store: S3BlockStore = config.open().await.unwrap(); + let admin = Database::connect(ConnectOptions::new(database_url.clone())) + .await + .unwrap(); + let schema = format!( + "tc735gc_{}_{}", + std::process::id(), + time::OffsetDateTime::now_utc().unix_timestamp_nanos() + ); + admin + .execute(Statement::from_string( + DbBackend::Postgres, + format!("CREATE SCHEMA {schema}"), + )) + .await + .unwrap(); + let mut options = ConnectOptions::new(database_url); + options.set_schema_search_path(schema.clone()); + let conn = Database::connect(options).await.unwrap(); + let _db = SpaceDatabase::new( + conn.clone(), + store.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let owner = Owner::generate().unwrap(); + store.create(&owner.space).await.unwrap(); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"tc735-orphan-cleanup").await.unwrap(); + let hash = stage.hash(); + tinycloud_core::kv_orphans::register_candidates(&conn, &[(owner.space.clone(), hash)]) + .await + .unwrap(); + assert_eq!( + ImmutableWriteStore::::persist(&store, &owner.space, stage) + .await + .unwrap(), + hash + ); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(20)); + assert_eq!( + tinycloud_core::kv_orphans::collect_candidates(&conn, &store, 1, time::Duration::ZERO,) + .await + .unwrap(), + 1 + ); + assert!(!store.contains(&owner.space, &hash).await.unwrap()); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); + // Simulate a known failure whose object removal succeeded but whose + // candidate-row deletion did not commit. A later GC pass must not debit + // the physical-byte counter a second time. + let mut failed = HashBuffer::new(Vec::new()); + failed.write_all(b"tc735-known-failure").await.unwrap(); + let failed_hash = failed.hash(); + let candidate = [(owner.space.clone(), failed_hash)]; + tinycloud_core::kv_orphans::register_candidates(&conn, &candidate) + .await + .unwrap(); + ImmutableWriteStore::::persist(&store, &owner.space, failed) + .await + .unwrap(); + tinycloud_core::kv_orphans::refund_failed(&conn, &store, &candidate) + .await + .unwrap(); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); + tinycloud_core::kv_orphans::register_candidates(&conn, &candidate) + .await + .unwrap(); + assert_eq!( + tinycloud_core::kv_orphans::collect_candidates(&conn, &store, 1, time::Duration::ZERO,) + .await + .unwrap(), + 0, + "the stale candidate has no physical object left to remove" + ); + assert!(!store.contains(&owner.space, &failed_hash).await.unwrap()); + assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); + conn.close().await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::Postgres, + format!("DROP SCHEMA {schema} CASCADE"), + )) + .await + .unwrap(); +} + async fn run_matrix( conn: DatabaseConnection, store: B, From 0e05eca3272a710092dc1bde66dd540dbf7f9cc3 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 21:58:38 +0000 Subject: [PATCH 02/17] Batch authorization graph snapshot in one SQL round trip --- CHANGELOG.md | 1 + tinycloud-core/src/auth_graph.rs | 643 ++++++++++++++++-- .../tests/postgres_round_trip_budget.rs | 17 +- 3 files changed, 612 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18907086..09ec6ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Load each invocation's complete authorization graph (proof edges, delegation windows and facts, abilities and caveats, revocations) in one SQL round trip on PostgreSQL, SQLite and MySQL rather than four, without changing Rust capability validation or schema (TC-735). - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/auth_graph.rs b/tinycloud-core/src/auth_graph.rs index 0fafaecd..99b21e94 100644 --- a/tinycloud-core/src/auth_graph.rs +++ b/tinycloud-core/src/auth_graph.rs @@ -1,20 +1,21 @@ //! TC-269: transaction-scoped authorization graph snapshot for the //! production invocation path. //! -//! Instead of walking `parent_delegations` one node at a time and issuing a -//! revocation lookup per ancestor (repeated for chain locking, revocation -//! checks, and chain-window validation), the snapshot batch-loads the whole -//! proof closure once: parent edges, then the -//! delegation rows, the closure's ability/caveat rows, and the closure's -//! revocations in one query each. All chain checks then run in memory against -//! the same consistent view. +//! Instead of walking `parent_delegations` one node at a time, a single +//! transaction-scoped query returns tagged parent edges, delegation rows, +//! ability/caveat rows, and revocations for the entire proof closure. Rust +//! reconstructs the same snapshot and performs all chain checks in memory +//! against that consistent view. use crate::hash::Hash; use crate::models::{abilities, delegation, revocation}; use crate::relationships::parent_delegations; use sea_orm::{ entity::prelude::*, - sea_query::{Alias, CommonTableExpression, Expr, JoinType, Query, UnionType, WithClause}, + sea_query::{ + Alias, CommonTableExpression, Expr, JoinType, Query, SelectStatement, SimpleExpr, + UnionType, WithClause, WithQuery, + }, ConnectionTrait, }; use std::collections::{HashMap, HashSet}; @@ -127,7 +128,291 @@ pub(crate) async fn load_closure_edges( Ok((nodes, edges)) } -#[derive(Debug)] +/// Each branch projects the same columns, with a distinct slot for each +/// underlying SQL type. In particular, JSON and text never share a UNION +/// slot, and the legacy model decoders still parse every field in Rust. +fn snapshot_branch(kind: i32, fields: [SimpleExpr; 12]) -> SelectStatement { + let mut query = Query::select(); + query.expr_as(Expr::val(kind), Alias::new("kind")); + for (name, expr) in [ + "node", + "parent", + "delegator", + "delegatee", + "expiry", + "issued_at", + "not_before", + "facts", + "serialization", + "resource", + "ability", + "caveats", + ] + .into_iter() + .zip(fields) + { + query.expr_as(expr, Alias::new(name)); + } + query +} + +fn snapshot_query(roots: &[Hash]) -> WithQuery { + let roots_name = Alias::new("proof_roots"); + let edges_name = Alias::new("authorization_edges"); + let bounded_name = Alias::new("bounded_edges"); + let nodes_name = Alias::new("graph_nodes"); + let id = Alias::new("id"); + let child = Alias::new("child"); + let parent = Alias::new("parent"); + + let mut root_query = Query::select(); + root_query.expr_as(Expr::val(roots[0]), id.clone()); + for root in roots.iter().skip(1) { + root_query.union( + UnionType::Distinct, + Query::select().expr(Expr::val(*root)).to_owned(), + ); + } + let roots_cte = CommonTableExpression::new() + .table_name(roots_name.clone()) + .column(id.clone()) + .query(root_query) + .to_owned(); + + let base = Query::select() + .columns([ + parent_delegations::Column::Child, + parent_delegations::Column::Parent, + ]) + .from(parent_delegations::Entity) + .and_where( + Expr::col(parent_delegations::Column::Child).in_subquery( + Query::select() + .column((roots_name.clone(), id.clone())) + .from(roots_name.clone()) + .to_owned(), + ), + ) + .to_owned(); + let recursive = Query::select() + .columns([ + ( + parent_delegations::Entity, + parent_delegations::Column::Child, + ), + ( + parent_delegations::Entity, + parent_delegations::Column::Parent, + ), + ]) + .from(parent_delegations::Entity) + .join( + JoinType::InnerJoin, + edges_name.clone(), + Expr::col(( + parent_delegations::Entity, + parent_delegations::Column::Child, + )) + .equals((edges_name.clone(), parent.clone())), + ) + .to_owned(); + let mut closure_query = base; + closure_query.union(UnionType::Distinct, recursive); + let edges_cte = CommonTableExpression::new() + .table_name(edges_name.clone()) + .columns([child.clone(), parent.clone()]) + .query(closure_query) + .to_owned(); + + let edge_cap = MAX_CHAIN_TRAVERSAL_NODES * MAX_CHAIN_TRAVERSAL_NODES; + let bounded_cte = CommonTableExpression::new() + .table_name(bounded_name.clone()) + .columns([child.clone(), parent.clone()]) + .query( + Query::select() + .columns([ + (edges_name.clone(), child.clone()), + (edges_name.clone(), parent.clone()), + ]) + .from(edges_name) + .limit((edge_cap + 1) as u64) + .to_owned(), + ) + .to_owned(); + let mut node_query = Query::select() + .column((roots_name.clone(), id.clone())) + .from(roots_name) + .to_owned(); + node_query.union( + UnionType::Distinct, + Query::select() + .column((bounded_name.clone(), child.clone())) + .from(bounded_name.clone()) + .to_owned(), + ); + node_query.union( + UnionType::Distinct, + Query::select() + .column((bounded_name.clone(), parent.clone())) + .from(bounded_name.clone()) + .to_owned(), + ); + let nodes_cte = CommonTableExpression::new() + .table_name(nodes_name.clone()) + .column(id.clone()) + .query(node_query) + .to_owned(); + let null = || Expr::cust("NULL"); + let field = |table: Alias, column: Alias| Expr::col((table, column)).into(); + let node = |table: Alias| Expr::col((table, id.clone())).into(); + let within_budget = || { + Expr::cust(format!( + "(SELECT COUNT(*) FROM graph_nodes) <= {} AND (SELECT COUNT(*) FROM bounded_edges) <= {}", + MAX_CHAIN_TRAVERSAL_NODES, edge_cap + )) + }; + // A zero-row type anchor precedes UNION ALL. PostgreSQL resolves an + // untyped NULL in the first branch to text before seeing later branches; + // selecting real columns here preserves timestamp/JSON/binary SQL types + // without backend-specific casts or an extra database round trip. + let mut query = snapshot_branch( + 4, + [ + Expr::col((delegation::Entity, delegation::Column::Id)).into(), + Expr::col(( + parent_delegations::Entity, + parent_delegations::Column::Parent, + )) + .into(), + Expr::col((delegation::Entity, delegation::Column::Delegator)).into(), + Expr::col((delegation::Entity, delegation::Column::Delegatee)).into(), + Expr::col((delegation::Entity, delegation::Column::Expiry)).into(), + Expr::col((delegation::Entity, delegation::Column::IssuedAt)).into(), + Expr::col((delegation::Entity, delegation::Column::NotBefore)).into(), + Expr::col((delegation::Entity, delegation::Column::Facts)).into(), + Expr::col((delegation::Entity, delegation::Column::Serialization)).into(), + Expr::col((abilities::Entity, abilities::Column::Resource)).into(), + Expr::col((abilities::Entity, abilities::Column::Ability)).into(), + Expr::col((abilities::Entity, abilities::Column::Caveats)).into(), + ], + ); + query + .from(delegation::Entity) + .from(parent_delegations::Entity) + .from(abilities::Entity) + .and_where(Expr::val(1).eq(0)); + let mut edge_rows = snapshot_branch( + 0, + [ + field(bounded_name.clone(), child), + field(bounded_name.clone(), parent), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + ], + ); + edge_rows.from(bounded_name.clone()); + query.union(UnionType::All, edge_rows); + let mut delegations = snapshot_branch( + 1, + [ + node(nodes_name.clone()), + null(), + Expr::col((delegation::Entity, delegation::Column::Delegator)).into(), + Expr::col((delegation::Entity, delegation::Column::Delegatee)).into(), + Expr::col((delegation::Entity, delegation::Column::Expiry)).into(), + Expr::col((delegation::Entity, delegation::Column::IssuedAt)).into(), + Expr::col((delegation::Entity, delegation::Column::NotBefore)).into(), + Expr::col((delegation::Entity, delegation::Column::Facts)).into(), + Expr::col((delegation::Entity, delegation::Column::Serialization)).into(), + null(), + null(), + null(), + ], + ); + delegations + .from(delegation::Entity) + .join( + JoinType::InnerJoin, + nodes_name.clone(), + Expr::col((delegation::Entity, delegation::Column::Id)) + .equals((nodes_name.clone(), id.clone())), + ) + .and_where(within_budget()); + query.union(UnionType::All, delegations); + let mut abilities_query = snapshot_branch( + 2, + [ + node(nodes_name.clone()), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + Expr::col((abilities::Entity, abilities::Column::Resource)).into(), + Expr::col((abilities::Entity, abilities::Column::Ability)).into(), + Expr::col((abilities::Entity, abilities::Column::Caveats)).into(), + ], + ); + abilities_query + .from(abilities::Entity) + .join( + JoinType::InnerJoin, + nodes_name.clone(), + Expr::col((abilities::Entity, abilities::Column::Delegation)) + .equals((nodes_name.clone(), id.clone())), + ) + .and_where(within_budget()); + query.union(UnionType::All, abilities_query); + let mut revocations = snapshot_branch( + 3, + [ + node(nodes_name.clone()), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + ], + ); + revocations + .from(revocation::Entity) + .join( + JoinType::InnerJoin, + nodes_name, + Expr::col((revocation::Entity, revocation::Column::Revoked)) + .equals((Alias::new("graph_nodes"), id)), + ) + .and_where(within_budget()); + query.union(UnionType::All, revocations); + query.with( + WithClause::new() + .recursive(true) + .cte(roots_cte) + .cte(edges_cte) + .cte(bounded_cte) + .cte(nodes_cte) + .to_owned(), + ) +} + +#[derive(Debug, PartialEq, Eq)] pub(crate) struct AuthGraphSnapshot { parents: HashMap>, delegations: HashMap, @@ -142,44 +427,92 @@ impl AuthGraphSnapshot { db: &C, roots: &[Hash], ) -> Result { - let (nodes, parents) = load_closure_edges(db, roots).await?; + let mut nodes = roots.to_vec(); + nodes.sort_by(|left, right| left.as_ref().cmp(right.as_ref())); + nodes.dedup(); + if nodes.len() > MAX_CHAIN_TRAVERSAL_NODES { + return Err(ChainTraversalError::LimitExceeded); + } if nodes.is_empty() { return Ok(Self { - parents, + parents: HashMap::new(), delegations: HashMap::new(), abilities: HashMap::new(), revoked: HashSet::new(), }); } - let delegations = delegation::Entity::find() - .filter(delegation::Column::Id.is_in(nodes.iter().copied())) - .all(db) - .await? - .into_iter() - .map(|row| (row.id, row)) - .collect(); - let mut ability_rows: HashMap> = HashMap::new(); - for row in abilities::Entity::find() - .filter(abilities::Column::Delegation.is_in(nodes.iter().copied())) - .all(db) - .await? - { - ability_rows.entry(row.delegation).or_default().push(row); + let rows = db + .query_all(db.get_database_backend().build(&snapshot_query(&nodes))) + .await?; + let mut parents: HashMap> = HashMap::new(); + let mut delegations = HashMap::new(); + let mut abilities: HashMap> = HashMap::new(); + let mut revoked = HashSet::new(); + let mut edge_count = 0; + for row in rows { + let node: Hash = row.try_get("", "node")?; + match row.try_get::("", "kind")? { + 0 => { + edge_count += 1; + if edge_count > MAX_CHAIN_TRAVERSAL_NODES * MAX_CHAIN_TRAVERSAL_NODES { + return Err(ChainTraversalError::LimitExceeded); + } + parents + .entry(node) + .or_default() + .push(row.try_get("", "parent")?); + } + 1 => { + delegations.insert( + node, + delegation::Model { + id: node, + delegator: row.try_get("", "delegator")?, + delegatee: row.try_get("", "delegatee")?, + expiry: row.try_get("", "expiry")?, + issued_at: row.try_get("", "issued_at")?, + not_before: row.try_get("", "not_before")?, + facts: row.try_get("", "facts")?, + serialization: row.try_get("", "serialization")?, + }, + ); + } + 2 => { + abilities.entry(node).or_default().push(abilities::Model { + delegation: node, + resource: row.try_get("", "resource")?, + ability: row.try_get("", "ability")?, + caveats: row.try_get("", "caveats")?, + }); + } + 3 => { + revoked.insert(node); + } + _ => return Err(DbErr::Type("unknown authorization graph row".into()).into()), + } + } + for ancestor_ids in parents.values_mut() { + ancestor_ids.sort_by(|left, right| left.as_ref().cmp(right.as_ref())); + ancestor_ids.dedup(); + } + let mut visited = nodes.iter().copied().collect::>(); + let mut frontier = nodes.clone(); + while let Some(current) = frontier.pop() { + for parent in parents.get(¤t).map(Vec::as_slice).unwrap_or(&[]) { + if visited.insert(*parent) { + if visited.len() > MAX_CHAIN_TRAVERSAL_NODES { + return Err(ChainTraversalError::LimitExceeded); + } + nodes.push(*parent); + frontier.push(*parent); + } + } } - - let revoked = revocation::Entity::find() - .filter(revocation::Column::Revoked.is_in(nodes.iter().copied())) - .all(db) - .await? - .into_iter() - .map(|row| row.revoked) - .collect(); - Ok(Self { parents, delegations, - abilities: ability_rows, + abilities, revoked, }) } @@ -282,9 +615,10 @@ mod tests { use crate::hash::hash; use crate::migrations::Migrator; use crate::models::actor; + use crate::types::{Caveats, Facts}; use sea_orm::{ ActiveModelTrait, ActiveValue::Set, ConnectOptions, Database, DatabaseConnection, - QueryFilter, + DbBackend, EntityTrait, QueryFilter, Statement, }; use sea_orm_migration::MigratorTrait; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -310,6 +644,235 @@ mod tests { (db, counter) } + // Frozen pre-batch implementation used only to compare complete snapshots. + async fn load_sequential( + db: &C, + roots: &[Hash], + ) -> Result { + let (nodes, parents) = load_closure_edges(db, roots).await?; + if nodes.is_empty() { + return Ok(AuthGraphSnapshot { + parents, + delegations: HashMap::new(), + abilities: HashMap::new(), + revoked: HashSet::new(), + }); + } + let delegations = delegation::Entity::find() + .filter(delegation::Column::Id.is_in(nodes.iter().copied())) + .all(db) + .await? + .into_iter() + .map(|row| (row.id, row)) + .collect(); + let mut ability_rows: HashMap> = HashMap::new(); + for row in abilities::Entity::find() + .filter(abilities::Column::Delegation.is_in(nodes.iter().copied())) + .all(db) + .await? + { + ability_rows.entry(row.delegation).or_default().push(row); + } + let revoked = revocation::Entity::find() + .filter(revocation::Column::Revoked.is_in(nodes.iter().copied())) + .all(db) + .await? + .into_iter() + .map(|row| row.revoked) + .collect(); + Ok(AuthGraphSnapshot { + parents, + delegations, + abilities: ability_rows, + revoked, + }) + } + + #[tokio::test] + async fn snapshot_load_is_one_round_trip_and_matches_sequential() { + let (db, counter) = counted_database().await; + let ids = insert_chain(&db, "one-trip", 2).await; + let expected = load_sequential(&db, &[ids[0]]).await.unwrap(); + let before = counter.load(Ordering::SeqCst); + let actual = AuthGraphSnapshot::load(&db, &[ids[0]]).await.unwrap(); + assert_eq!(actual, expected); + assert_eq!(counter.load(Ordering::SeqCst) - before, 1); + } + + async fn assert_fixture_equivalence(db: &DatabaseConnection) { + actor::ActiveModel { + id: Set("did:key:actor".to_string()), + } + .insert(db) + .await + .unwrap(); + actor::ActiveModel { + id: Set("did:key:alias".to_string()), + } + .insert(db) + .await + .unwrap(); + let ids = insert_chain(db, "cross-backend", 3).await; + let alias = hash(b"cross-backend-alias"); + insert_delegation(db, alias).await; + insert_edge(db, alias, ids[2]).await; + insert_edge(db, ids[0], ids[2]).await; + + let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap(); + let mut windowed: delegation::ActiveModel = delegation::Entity::find_by_id(ids[1]) + .one(db) + .await + .unwrap() + .unwrap() + .into(); + windowed.delegator = Set("did:key:alias".to_string()); + windowed.issued_at = Set(Some(now - time::Duration::hours(1))); + windowed.not_before = Set(Some(now - time::Duration::minutes(15))); + windowed.expiry = Set(Some(now + time::Duration::hours(1))); + windowed.facts = Set(Some(Facts( + [( + "alias".to_string(), + serde_json::json!({"principal": "did:key:alias"}), + )] + .into(), + ))); + windowed.update(db).await.unwrap(); + + for id in [ids[0], ids[2], alias] { + abilities::ActiveModel { + delegation: Set(id), + resource: Set("tinycloud:did:key:actor:files/kv/doc".parse().unwrap()), + ability: Set("tinycloud.kv/get".to_string().try_into().unwrap()), + caveats: Set(Caveats( + [("limit".to_string(), serde_json::json!({"max": 7}))].into(), + )), + } + .insert(db) + .await + .unwrap(); + } + revocation::ActiveModel { + id: Set(hash(b"cross-backend-revocation")), + revoker: Set("did:key:actor".to_string()), + revoked: Set(ids[2]), + serialization: Set(b"revoked".to_vec()), + revoked_at: Set(Some(now)), + } + .insert(db) + .await + .unwrap(); + + for roots in [ + vec![ids[0]], + vec![alias, ids[0], alias], + vec![hash(b"missing-proof"), ids[1]], + ] { + let before = load_sequential(db, &roots).await.unwrap(); + let after = AuthGraphSnapshot::load(db, &roots).await.unwrap(); + assert_eq!(after, before, "roots {roots:?}"); + } + assert_eq!( + AuthGraphSnapshot::load(db, &[]).await.unwrap(), + load_sequential(db, &[]).await.unwrap() + ); + } + + #[tokio::test] + async fn sqlite_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + assert_fixture_equivalence(&db).await; + } + + #[tokio::test] + async fn postgres_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases", + ) + .await + else { + return; + }; + assert_fixture_equivalence(&fixture.conn).await; + } + + struct MySqlGraphFixture { + url: String, + name: String, + } + + impl Drop for MySqlGraphFixture { + fn drop(&mut self) { + let (url, name) = (self.url.clone(), self.name.clone()); + let result = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + Database::connect(url) + .await? + .execute(Statement::from_string( + DbBackend::MySql, + format!("DROP DATABASE IF EXISTS {name}"), + )) + .await + .map(|_| ()) + }) + }) + .join(); + if !matches!(result, Ok(Ok(()))) { + eprintln!("failed to drop isolated MySQL graph database {}", self.name); + } + } + } + + #[tokio::test] + async fn mysql_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { + let Ok(url) = std::env::var("TINYCLOUD_TEST_MYSQL_URL") else { + eprintln!("skipping MySQL graph snapshot equivalence (TINYCLOUD_TEST_MYSQL_URL unset)"); + return; + }; + let (prefix, _) = url + .rsplit_once('/') + .expect("MySQL URL must include a database"); + let name = format!( + "tc_graph_{}_{}", + std::process::id(), + OffsetDateTime::now_utc().unix_timestamp_nanos() + ); + let admin = Database::connect(url.clone()).await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::MySql, + format!("CREATE DATABASE {name}"), + )) + .await + .unwrap(); + let _fixture = MySqlGraphFixture { + url: url.clone(), + name: name.clone(), + }; + let db = Database::connect(format!("{prefix}/{name}")).await.unwrap(); + // The full repository migrator predates MySQL's indexed-column length + // limits. Keep this fixture scoped to the four auth-graph model tables + // rather than changing unrelated migrations for a query equivalence test. + for ddl in [ + "CREATE TABLE actor (id VARCHAR(191) PRIMARY KEY)", + "CREATE TABLE delegation (id VARBINARY(34) PRIMARY KEY, delegator VARCHAR(191) NOT NULL, delegatee VARCHAR(191) NOT NULL, expiry DATETIME NULL, issued_at DATETIME NULL, not_before DATETIME NULL, facts JSON NULL, serialization BLOB NOT NULL)", + "CREATE TABLE parent_delegation (child VARBINARY(34) NOT NULL, parent VARBINARY(34) NOT NULL, PRIMARY KEY (parent, child))", + "CREATE TABLE ability (resource VARCHAR(255) NOT NULL, ability VARCHAR(255) NOT NULL, delegation VARBINARY(34) NOT NULL, caveats JSON NOT NULL, PRIMARY KEY (resource, ability, delegation))", + "CREATE TABLE revocation (id VARBINARY(34) PRIMARY KEY, revoker VARCHAR(191) NOT NULL, revoked VARBINARY(34) NOT NULL, serialization BLOB NOT NULL, revoked_at DATETIME NULL)", + ] { + db.execute(Statement::from_string(DbBackend::MySql, ddl.to_string())) + .await + .unwrap(); + } + assert_fixture_equivalence(&db).await; + db.close().await.unwrap(); + admin.close().await.unwrap(); + } + async fn insert_delegation(db: &DatabaseConnection, id: Hash) { delegation::ActiveModel { id: Set(id), @@ -464,9 +1027,7 @@ mod tests { /// depth-amplified number of queries (per-node closure walks repeated for /// locking, revocation, and chain-window checks, plus a revocation lookup /// per ancestor). The new path uses one pre-transaction closure query to - /// establish the lock set, then one transaction-scoped snapshot consisting - /// of one closure query and one batch each for delegations, abilities, and - /// revocations. + /// establish the lock set, then one tagged transaction-scoped snapshot query. #[tokio::test] async fn snapshot_query_batches_are_bounded_versus_depth_amplified_traversal() { for (depth, expected_legacy) in [(0, 6), (1, 10), (4, 22)] { @@ -502,7 +1063,7 @@ mod tests { let legacy_queries = counter.load(Ordering::SeqCst) - before; // New production path: one recursive edge query to establish the - // lock set, then a four-query transaction-scoped snapshot. + // lock set, then one transaction-scoped snapshot query. let before = counter.load(Ordering::SeqCst); let (lock_keys, _) = load_closure_edges(&db, &[leaf]).await.unwrap(); let snapshot = AuthGraphSnapshot::load(&db, &[leaf]).await.unwrap(); @@ -511,7 +1072,7 @@ mod tests { assert_eq!(lock_keys.len(), depth + 1); assert_eq!(snapshot.chain_ids_from(&leaf).len(), depth + 1); assert_eq!(legacy_queries, expected_legacy, "depth {depth}"); - assert_eq!(optimized_queries, 5, "depth {depth}"); + assert_eq!(optimized_queries, 2, "depth {depth}"); } } } diff --git a/tinycloud-node-server/tests/postgres_round_trip_budget.rs b/tinycloud-node-server/tests/postgres_round_trip_budget.rs index 95e33b00..dc7c4650 100644 --- a/tinycloud-node-server/tests/postgres_round_trip_budget.rs +++ b/tinycloud-node-server/tests/postgres_round_trip_budget.rs @@ -113,15 +113,16 @@ use tokio::{ net::{TcpListener, TcpStream}, }; -/// Round trips allowed per flow; see the module docs. Measured with -/// `test_before_acquire(false)`, database identity resolve as one statement, -/// the read audit as one statement and no second sequence lock (TC-951): +/// Round trips allowed per flow; see the module docs. Each invocation's +/// transaction-scoped authorization snapshot now fetches edges, delegations, +/// abilities and revocations in one tagged query (TC-735); capability checks +/// and connection-release pings are unchanged. Measured on PostgreSQL: /// -/// - sign-in 75 = peer key 0 + host `/delegate` 17 + session `/delegate` 18 -/// + account-space SQL write 40; -/// - kv put 37; -/// - kv get 22, including the read-audit commit. -const BUDGETS: [(&str, u64); 3] = [("sign-in", 75), ("kv put", 37), ("kv get", 22)]; +/// - sign-in 69 = peer key 0 + host `/delegate` 17 + session `/delegate` 18 +/// + account-space SQL write 34; +/// - kv put 34; +/// - kv get 16, including the read-audit commit. +const BUDGETS: [(&str, u64); 3] = [("sign-in", 69), ("kv put", 34), ("kv get", 16)]; /// Full sign-in plus put plus get rounds run before the measured one. const WARM_UP_ROUNDS: u8 = 3; From 082393bdab5033a75db08d3b1d8c3e36b7a7e745 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 22:23:10 +0000 Subject: [PATCH 03/17] Bound auth graph lookups and canonicalize ability diagnostics --- CHANGELOG.md | 2 +- tinycloud-core/src/auth_graph.rs | 218 ++++++++++++++++++++++++++----- 2 files changed, 183 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ec6ced..205e315b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- Load each invocation's complete authorization graph (proof edges, delegation windows and facts, abilities and caveats, revocations) in one SQL round trip on PostgreSQL, SQLite and MySQL rather than four, without changing Rust capability validation or schema (TC-735). +- Load each invocation's authorization graph in one SQL round trip on PostgreSQL, SQLite and MySQL rather than four. Bound metadata lookups to 65 nodes for indexed access on large tables while the unchanged 64-node authorization limit still denies oversized chains. Sort abilities by `(resource, ability)` on every backend; authorization decisions are unchanged, but the diagnostic reason when multiple caveats all reject may normalize. No schema change (TC-735). - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/auth_graph.rs b/tinycloud-core/src/auth_graph.rs index 99b21e94..1ce161e6 100644 --- a/tinycloud-core/src/auth_graph.rs +++ b/tinycloud-core/src/auth_graph.rs @@ -161,6 +161,7 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { let edges_name = Alias::new("authorization_edges"); let bounded_name = Alias::new("bounded_edges"); let nodes_name = Alias::new("graph_nodes"); + let lookup_name = Alias::new("lookup_nodes"); let id = Alias::new("id"); let child = Alias::new("child"); let parent = Alias::new("parent"); @@ -262,6 +263,20 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { .column(id.clone()) .query(node_query) .to_owned(); + // The authorization limit is 64 nodes. Keep one sentinel in the planner- + // visible metadata input; within_budget still counts every node from the + // edge-bounded graph_nodes result, so truncation cannot authorize a chain. + let lookup_cte = CommonTableExpression::new() + .table_name(lookup_name.clone()) + .column(id.clone()) + .query( + Query::select() + .column((nodes_name.clone(), id.clone())) + .from(nodes_name.clone()) + .limit((MAX_CHAIN_TRAVERSAL_NODES + 1) as u64) + .to_owned(), + ) + .to_owned(); let null = || Expr::cust("NULL"); let field = |table: Alias, column: Alias| Expr::col((table, column)).into(); let node = |table: Alias| Expr::col((table, id.clone())).into(); @@ -323,7 +338,7 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { let mut delegations = snapshot_branch( 1, [ - node(nodes_name.clone()), + node(lookup_name.clone()), null(), Expr::col((delegation::Entity, delegation::Column::Delegator)).into(), Expr::col((delegation::Entity, delegation::Column::Delegatee)).into(), @@ -341,16 +356,16 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { .from(delegation::Entity) .join( JoinType::InnerJoin, - nodes_name.clone(), + lookup_name.clone(), Expr::col((delegation::Entity, delegation::Column::Id)) - .equals((nodes_name.clone(), id.clone())), + .equals((lookup_name.clone(), id.clone())), ) .and_where(within_budget()); query.union(UnionType::All, delegations); let mut abilities_query = snapshot_branch( 2, [ - node(nodes_name.clone()), + node(lookup_name.clone()), null(), null(), null(), @@ -368,16 +383,16 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { .from(abilities::Entity) .join( JoinType::InnerJoin, - nodes_name.clone(), + lookup_name.clone(), Expr::col((abilities::Entity, abilities::Column::Delegation)) - .equals((nodes_name.clone(), id.clone())), + .equals((lookup_name.clone(), id.clone())), ) .and_where(within_budget()); query.union(UnionType::All, abilities_query); let mut revocations = snapshot_branch( 3, [ - node(nodes_name.clone()), + node(lookup_name.clone()), null(), null(), null(), @@ -395,9 +410,9 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { .from(revocation::Entity) .join( JoinType::InnerJoin, - nodes_name, + lookup_name, Expr::col((revocation::Entity, revocation::Column::Revoked)) - .equals((Alias::new("graph_nodes"), id)), + .equals((Alias::new("lookup_nodes"), id)), ) .and_where(within_budget()); query.union(UnionType::All, revocations); @@ -408,10 +423,24 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { .cte(edges_cte) .cte(bounded_cte) .cte(nodes_cte) + .cte(lookup_cte) .to_owned(), ) } +// The old metadata query had no ORDER BY. Authorization accepts any matching +// ability, but the last rejected caveat supplies a diagnostic reason. Use the +// typed `(resource, ability)` order on every backend; only that diagnostic can +// differ from the old backend-dependent order. +fn sort_abilities(abilities: &mut HashMap>) { + for rows in abilities.values_mut() { + rows.sort_unstable_by(|left, right| { + left.resource + .cmp(&right.resource) + .then_with(|| left.ability.cmp(&right.ability)) + }); + } +} #[derive(Debug, PartialEq, Eq)] pub(crate) struct AuthGraphSnapshot { parents: HashMap>, @@ -496,6 +525,7 @@ impl AuthGraphSnapshot { ancestor_ids.sort_by(|left, right| left.as_ref().cmp(right.as_ref())); ancestor_ids.dedup(); } + sort_abilities(&mut abilities); let mut visited = nodes.iter().copied().collect::>(); let mut frontier = nodes.clone(); while let Some(current) = frontier.pop() { @@ -673,6 +703,9 @@ mod tests { { ability_rows.entry(row.delegation).or_default().push(row); } + // Compare the old loader's rows under the new explicit order, rather + // than accidentally pinning one backend's unspecified row order. + sort_abilities(&mut ability_rows); let revoked = revocation::Entity::find() .filter(revocation::Column::Revoked.is_in(nodes.iter().copied())) .all(db) @@ -738,18 +771,26 @@ mod tests { ))); windowed.update(db).await.unwrap(); + // Deliberately insert away from canonical `(resource, ability)` order. + // A one-ability fixture cannot detect changed rejection-reason order. for id in [ids[0], ids[2], alias] { - abilities::ActiveModel { - delegation: Set(id), - resource: Set("tinycloud:did:key:actor:files/kv/doc".parse().unwrap()), - ability: Set("tinycloud.kv/get".to_string().try_into().unwrap()), - caveats: Set(Caveats( - [("limit".to_string(), serde_json::json!({"max": 7}))].into(), - )), + for (index, action) in ["put", "get", "list", "del"].iter().enumerate() { + for suffix in ["z", "b", "a"] { + abilities::ActiveModel { + delegation: Set(id), + resource: Set(format!("tinycloud:did:key:actor:files/kv/{suffix}") + .parse() + .unwrap()), + ability: Set(format!("tinycloud.kv/{action}").try_into().unwrap()), + caveats: Set(Caveats( + [("limit".to_string(), serde_json::json!({"max": index + 1}))].into(), + )), + } + .insert(db) + .await + .unwrap(); + } } - .insert(db) - .await - .unwrap(); } revocation::ActiveModel { id: Set(hash(b"cross-backend-revocation")), @@ -777,11 +818,129 @@ mod tests { ); } + async fn assert_canonical_caveat_reason(db: &DatabaseConnection) { + let id = hash(b"canonical-overlapping-caveats"); + delegation::ActiveModel { + id: Set(id), + delegator: Set("did:key:actor".to_string()), + delegatee: Set("did:key:alias".to_string()), + expiry: Set(None), + issued_at: Set(None), + not_before: Set(None), + facts: Set(None), + serialization: Set(id.as_ref().to_vec()), + } + .insert(db) + .await + .unwrap(); + let sql_caveats = Caveats( + [( + "0".to_string(), + serde_json::json!({ + "mode": "constrained-statements", + "readOnly": true, + "statements": [ + {"name": "get", "sql": "SELECT 1", "fixedParams": []} + ] + }), + )] + .into(), + ); + for (resource, caveats) in [ + ( + "tinycloud:did:key:actor:files/kv", + Caveats([("tenant".to_string(), serde_json::json!("a"))].into()), + ), + ("tinycloud:did:key:actor:files/kv/doc", sql_caveats.clone()), + ] { + abilities::ActiveModel { + delegation: Set(id), + resource: Set(resource.parse().unwrap()), + ability: Set("tinycloud.kv/get".to_string().try_into().unwrap()), + caveats: Set(caveats), + } + .insert(db) + .await + .unwrap(); + } + let snapshot = AuthGraphSnapshot::load(db, &[id]).await.unwrap(); + let rows = snapshot.abilities(&id); + assert_eq!(rows.len(), 2); + assert!( + rows[0].resource < rows[1].resource, + "canonical resource order" + ); + + let capability = crate::util::Capability { + resource: "tinycloud:did:key:actor:files/kv/doc".parse().unwrap(), + ability: "tinycloud.kv/get".to_string().try_into().unwrap(), + caveats: Caveats::default(), + }; + let parent = id.to_cid(0x55); + let err = crate::models::invocation::authorize_holder( + db, + "did:key:alias", + std::slice::from_ref(&capability), + &[parent], + OffsetDateTime::now_utc(), + ) + .await + .unwrap_err(); + assert!( + matches!( + &err, + crate::models::invocation::Error::InvalidInvocation( + crate::models::invocation::InvocationError::CaveatsNotContained(reason) + ) if reason == "containment-caveat-required" + ), + "unexpected canonical rejection: {err:?}" + ); + + // A matching constrained caveat still authorizes; sorting changes + // only which diagnostic wins if *all* candidates reject. + crate::models::invocation::authorize_holder( + db, + "did:key:alias", + &[crate::util::Capability { + caveats: sql_caveats, + ..capability + }], + &[parent], + OffsetDateTime::now_utc(), + ) + .await + .unwrap(); + } + + async fn assert_closure_limit(db: &DatabaseConnection) { + for size in [MAX_CHAIN_TRAVERSAL_NODES + 1, MAX_CHAIN_TRAVERSAL_NODES + 2] { + let ids = insert_chain(db, &format!("too-deep-{size}"), size - 1).await; + assert!(matches!( + AuthGraphSnapshot::load(db, &[ids[0]]).await, + Err(ChainTraversalError::LimitExceeded) + )); + } + + let ids = insert_chain(db, "boundary", MAX_CHAIN_TRAVERSAL_NODES - 1).await; + let snapshot = AuthGraphSnapshot::load(db, &[ids[0]]).await.unwrap(); + assert_eq!( + snapshot.chain_ids_from(&ids[0]).len(), + MAX_CHAIN_TRAVERSAL_NODES + ); + assert!( + snapshot.delegation(ids.last().unwrap()).is_some(), + "the 64th node must not be silently truncated" + ); + assert_eq!(snapshot, load_sequential(db, &[ids[0]]).await.unwrap()); + } + #[tokio::test] async fn sqlite_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { let db = Database::connect("sqlite::memory:").await.unwrap(); Migrator::up(&db, None).await.unwrap(); assert_fixture_equivalence(&db).await; + assert_canonical_caveat_reason(&db).await; + assert_closure_limit(&db).await; } #[tokio::test] @@ -794,6 +953,8 @@ mod tests { return; }; assert_fixture_equivalence(&fixture.conn).await; + assert_canonical_caveat_reason(&fixture.conn).await; + assert_closure_limit(&fixture.conn).await; } struct MySqlGraphFixture { @@ -869,6 +1030,8 @@ mod tests { .unwrap(); } assert_fixture_equivalence(&db).await; + assert_canonical_caveat_reason(&db).await; + assert_closure_limit(&db).await; db.close().await.unwrap(); admin.close().await.unwrap(); } @@ -934,23 +1097,6 @@ mod tests { } } - #[tokio::test] - async fn snapshot_load_fails_closed_at_traversal_limit() { - let (db, _) = counted_database().await; - let ids = insert_chain(&db, "deep", MAX_CHAIN_TRAVERSAL_NODES).await; - assert!(matches!( - AuthGraphSnapshot::load(&db, &[ids[0]]).await, - Err(ChainTraversalError::LimitExceeded) - )); - - let boundary = insert_chain(&db, "boundary", MAX_CHAIN_TRAVERSAL_NODES - 1).await; - let snapshot = AuthGraphSnapshot::load(&db, &[boundary[0]]).await.unwrap(); - assert_eq!( - snapshot.chain_ids_from(&boundary[0]).len(), - MAX_CHAIN_TRAVERSAL_NODES - ); - } - #[tokio::test] async fn snapshot_reports_direct_revocation() { let (db, _) = counted_database().await; From 01f7f25aceb441e57c3d86993e60f7241f0009aa Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 22:40:22 +0000 Subject: [PATCH 04/17] Protect pre-persist writes through freeze acknowledgement --- CHANGELOG.md | 3 +- docs/kv-sync.md | 13 + docs/meeting-publication-migration.md | 7 +- tinycloud-core/src/auth_graph.rs | 823 ++++++++++++++++-- tinycloud-core/src/db.rs | 670 ++++++++++++-- tinycloud-core/src/db_pool.rs | 33 +- tinycloud-core/src/kv_orphans.rs | 28 +- .../tests/postgres_connection_recovery.rs | 118 +++ .../tests/postgres_round_trip_budget.rs | 25 +- 9 files changed, 1588 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60147722..f883b780 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## [Unreleased] -- KV puts persist content-addressed blobs before taking the per-space sequence lock, while the lock still orders committed `kv/sync` cursors. A durable hash-fenced candidate table (`m20261015_020000_kv_orphan_candidates`) supports immediate cleanup and physical-byte quota refunds after known failures; a 60-second collector reclaims crash leftovers after a five-minute grace. Quota remains conservative and per process, not a durable global ledger (TC-735). +- KV puts persist content-addressed blobs before taking the per-space sequence lock, while the lock still orders committed `kv/sync` cursors. Protected legacy meeting writes acquire the durable freeze barrier before upload, so a freeze waits for their commit or abort and fails after 30 seconds if stalled. A durable hash-fenced candidate table (`m20261015_020000_kv_orphan_candidates`) supports immediate cleanup and physical-byte quota refunds after known failures; a 60-second collector reclaims crash leftovers after a five-minute grace. Quota remains conservative and per process, not a durable global ledger (TC-735). + Lost connections during candidate registration are retried idempotently; before any upload, a lost hash/freeze-lock transaction can restart once with both fences. Neither an upload nor a commit is replayed on an uncertain outcome. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/docs/kv-sync.md b/docs/kv-sync.md index d09fc408..06e24760 100644 --- a/docs/kv-sync.md +++ b/docs/kv-sync.md @@ -140,6 +140,19 @@ the lock through commit, so `seq` and sync cursors remain commit-ordered even when same-space uploads finish out of order. An unauthorized write uploads nothing. +For protected legacy meeting paths, the durable freeze row is locked before +upload and held until commit or abort. That barrier is distinct from the +per-space sequence lock: a freeze drains uploads already in flight, while +writes arriving after the freeze are rejected before upload. A stalled writer +causes the freeze attempt to fail after 30 seconds rather than acknowledge an +incomplete drain. + +Before upload, a lost pooled connection retries the candidate registration +with the same idempotent upsert. If the hash or freeze row lock loses its +transaction, the node restarts that transaction once and retakes both fences +before persisting anything. It never replays an upload or committed write +after a connection loss. + A failed upload, precondition, or database write triggers an immediate hash-fenced reference check. Unreferenced blobs are removed and their physical-byte quota is refunded before the failure returns. Historical diff --git a/docs/meeting-publication-migration.md b/docs/meeting-publication-migration.md index c665f242..6eca6582 100644 --- a/docs/meeting-publication-migration.md +++ b/docs/meeting-publication-migration.md @@ -48,7 +48,12 @@ An existing pause continues to block legacy writes under `xyz.tinycloud.tinychat Reads, chat keys, cursors, credentials, and other spaces retain their existing behavior. Frozen ordinary KV put/delete returns HTTP 409. The core guard also rejects internal legacy cleanup. The pause survives a node restart; only an exact generation-checked release removes it. -The durable guard remains locked by protected KV mutation transactions through storage persistence and commit. PostgreSQL row locking and SQLite writer serialization retain their existing behavior. Failed guard reads fail closed. Existing snapshot keys remain protected after the legacy pause is released. +The durable guard is acquired before a protected KV upload starts and stays locked through storage persistence and commit or abort. A freeze acknowledgement therefore drains writes already uploading; a later writer is refused before uploading. If a writer stalls, the freeze attempt fails after 30 seconds rather than acknowledging an incomplete drain. PostgreSQL row locking and SQLite writer serialization retain their existing behavior. Failed guard reads fail closed. Existing snapshot keys remain protected after the legacy pause is released. + +The acknowledgement covers DB commit or rollback and the end of any protected +upload, not physical orphan reclamation. Cleanup of a rolled-back write may +finish afterward; it can only delete and refund an unreferenced, hash-fenced +blob, never change committed KV rows or upload new data. ## Deployment compatibility diff --git a/tinycloud-core/src/auth_graph.rs b/tinycloud-core/src/auth_graph.rs index 0fafaecd..1ce161e6 100644 --- a/tinycloud-core/src/auth_graph.rs +++ b/tinycloud-core/src/auth_graph.rs @@ -1,20 +1,21 @@ //! TC-269: transaction-scoped authorization graph snapshot for the //! production invocation path. //! -//! Instead of walking `parent_delegations` one node at a time and issuing a -//! revocation lookup per ancestor (repeated for chain locking, revocation -//! checks, and chain-window validation), the snapshot batch-loads the whole -//! proof closure once: parent edges, then the -//! delegation rows, the closure's ability/caveat rows, and the closure's -//! revocations in one query each. All chain checks then run in memory against -//! the same consistent view. +//! Instead of walking `parent_delegations` one node at a time, a single +//! transaction-scoped query returns tagged parent edges, delegation rows, +//! ability/caveat rows, and revocations for the entire proof closure. Rust +//! reconstructs the same snapshot and performs all chain checks in memory +//! against that consistent view. use crate::hash::Hash; use crate::models::{abilities, delegation, revocation}; use crate::relationships::parent_delegations; use sea_orm::{ entity::prelude::*, - sea_query::{Alias, CommonTableExpression, Expr, JoinType, Query, UnionType, WithClause}, + sea_query::{ + Alias, CommonTableExpression, Expr, JoinType, Query, SelectStatement, SimpleExpr, + UnionType, WithClause, WithQuery, + }, ConnectionTrait, }; use std::collections::{HashMap, HashSet}; @@ -127,7 +128,320 @@ pub(crate) async fn load_closure_edges( Ok((nodes, edges)) } -#[derive(Debug)] +/// Each branch projects the same columns, with a distinct slot for each +/// underlying SQL type. In particular, JSON and text never share a UNION +/// slot, and the legacy model decoders still parse every field in Rust. +fn snapshot_branch(kind: i32, fields: [SimpleExpr; 12]) -> SelectStatement { + let mut query = Query::select(); + query.expr_as(Expr::val(kind), Alias::new("kind")); + for (name, expr) in [ + "node", + "parent", + "delegator", + "delegatee", + "expiry", + "issued_at", + "not_before", + "facts", + "serialization", + "resource", + "ability", + "caveats", + ] + .into_iter() + .zip(fields) + { + query.expr_as(expr, Alias::new(name)); + } + query +} + +fn snapshot_query(roots: &[Hash]) -> WithQuery { + let roots_name = Alias::new("proof_roots"); + let edges_name = Alias::new("authorization_edges"); + let bounded_name = Alias::new("bounded_edges"); + let nodes_name = Alias::new("graph_nodes"); + let lookup_name = Alias::new("lookup_nodes"); + let id = Alias::new("id"); + let child = Alias::new("child"); + let parent = Alias::new("parent"); + + let mut root_query = Query::select(); + root_query.expr_as(Expr::val(roots[0]), id.clone()); + for root in roots.iter().skip(1) { + root_query.union( + UnionType::Distinct, + Query::select().expr(Expr::val(*root)).to_owned(), + ); + } + let roots_cte = CommonTableExpression::new() + .table_name(roots_name.clone()) + .column(id.clone()) + .query(root_query) + .to_owned(); + + let base = Query::select() + .columns([ + parent_delegations::Column::Child, + parent_delegations::Column::Parent, + ]) + .from(parent_delegations::Entity) + .and_where( + Expr::col(parent_delegations::Column::Child).in_subquery( + Query::select() + .column((roots_name.clone(), id.clone())) + .from(roots_name.clone()) + .to_owned(), + ), + ) + .to_owned(); + let recursive = Query::select() + .columns([ + ( + parent_delegations::Entity, + parent_delegations::Column::Child, + ), + ( + parent_delegations::Entity, + parent_delegations::Column::Parent, + ), + ]) + .from(parent_delegations::Entity) + .join( + JoinType::InnerJoin, + edges_name.clone(), + Expr::col(( + parent_delegations::Entity, + parent_delegations::Column::Child, + )) + .equals((edges_name.clone(), parent.clone())), + ) + .to_owned(); + let mut closure_query = base; + closure_query.union(UnionType::Distinct, recursive); + let edges_cte = CommonTableExpression::new() + .table_name(edges_name.clone()) + .columns([child.clone(), parent.clone()]) + .query(closure_query) + .to_owned(); + + let edge_cap = MAX_CHAIN_TRAVERSAL_NODES * MAX_CHAIN_TRAVERSAL_NODES; + let bounded_cte = CommonTableExpression::new() + .table_name(bounded_name.clone()) + .columns([child.clone(), parent.clone()]) + .query( + Query::select() + .columns([ + (edges_name.clone(), child.clone()), + (edges_name.clone(), parent.clone()), + ]) + .from(edges_name) + .limit((edge_cap + 1) as u64) + .to_owned(), + ) + .to_owned(); + let mut node_query = Query::select() + .column((roots_name.clone(), id.clone())) + .from(roots_name) + .to_owned(); + node_query.union( + UnionType::Distinct, + Query::select() + .column((bounded_name.clone(), child.clone())) + .from(bounded_name.clone()) + .to_owned(), + ); + node_query.union( + UnionType::Distinct, + Query::select() + .column((bounded_name.clone(), parent.clone())) + .from(bounded_name.clone()) + .to_owned(), + ); + let nodes_cte = CommonTableExpression::new() + .table_name(nodes_name.clone()) + .column(id.clone()) + .query(node_query) + .to_owned(); + // The authorization limit is 64 nodes. Keep one sentinel in the planner- + // visible metadata input; within_budget still counts every node from the + // edge-bounded graph_nodes result, so truncation cannot authorize a chain. + let lookup_cte = CommonTableExpression::new() + .table_name(lookup_name.clone()) + .column(id.clone()) + .query( + Query::select() + .column((nodes_name.clone(), id.clone())) + .from(nodes_name.clone()) + .limit((MAX_CHAIN_TRAVERSAL_NODES + 1) as u64) + .to_owned(), + ) + .to_owned(); + let null = || Expr::cust("NULL"); + let field = |table: Alias, column: Alias| Expr::col((table, column)).into(); + let node = |table: Alias| Expr::col((table, id.clone())).into(); + let within_budget = || { + Expr::cust(format!( + "(SELECT COUNT(*) FROM graph_nodes) <= {} AND (SELECT COUNT(*) FROM bounded_edges) <= {}", + MAX_CHAIN_TRAVERSAL_NODES, edge_cap + )) + }; + // A zero-row type anchor precedes UNION ALL. PostgreSQL resolves an + // untyped NULL in the first branch to text before seeing later branches; + // selecting real columns here preserves timestamp/JSON/binary SQL types + // without backend-specific casts or an extra database round trip. + let mut query = snapshot_branch( + 4, + [ + Expr::col((delegation::Entity, delegation::Column::Id)).into(), + Expr::col(( + parent_delegations::Entity, + parent_delegations::Column::Parent, + )) + .into(), + Expr::col((delegation::Entity, delegation::Column::Delegator)).into(), + Expr::col((delegation::Entity, delegation::Column::Delegatee)).into(), + Expr::col((delegation::Entity, delegation::Column::Expiry)).into(), + Expr::col((delegation::Entity, delegation::Column::IssuedAt)).into(), + Expr::col((delegation::Entity, delegation::Column::NotBefore)).into(), + Expr::col((delegation::Entity, delegation::Column::Facts)).into(), + Expr::col((delegation::Entity, delegation::Column::Serialization)).into(), + Expr::col((abilities::Entity, abilities::Column::Resource)).into(), + Expr::col((abilities::Entity, abilities::Column::Ability)).into(), + Expr::col((abilities::Entity, abilities::Column::Caveats)).into(), + ], + ); + query + .from(delegation::Entity) + .from(parent_delegations::Entity) + .from(abilities::Entity) + .and_where(Expr::val(1).eq(0)); + let mut edge_rows = snapshot_branch( + 0, + [ + field(bounded_name.clone(), child), + field(bounded_name.clone(), parent), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + ], + ); + edge_rows.from(bounded_name.clone()); + query.union(UnionType::All, edge_rows); + let mut delegations = snapshot_branch( + 1, + [ + node(lookup_name.clone()), + null(), + Expr::col((delegation::Entity, delegation::Column::Delegator)).into(), + Expr::col((delegation::Entity, delegation::Column::Delegatee)).into(), + Expr::col((delegation::Entity, delegation::Column::Expiry)).into(), + Expr::col((delegation::Entity, delegation::Column::IssuedAt)).into(), + Expr::col((delegation::Entity, delegation::Column::NotBefore)).into(), + Expr::col((delegation::Entity, delegation::Column::Facts)).into(), + Expr::col((delegation::Entity, delegation::Column::Serialization)).into(), + null(), + null(), + null(), + ], + ); + delegations + .from(delegation::Entity) + .join( + JoinType::InnerJoin, + lookup_name.clone(), + Expr::col((delegation::Entity, delegation::Column::Id)) + .equals((lookup_name.clone(), id.clone())), + ) + .and_where(within_budget()); + query.union(UnionType::All, delegations); + let mut abilities_query = snapshot_branch( + 2, + [ + node(lookup_name.clone()), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + Expr::col((abilities::Entity, abilities::Column::Resource)).into(), + Expr::col((abilities::Entity, abilities::Column::Ability)).into(), + Expr::col((abilities::Entity, abilities::Column::Caveats)).into(), + ], + ); + abilities_query + .from(abilities::Entity) + .join( + JoinType::InnerJoin, + lookup_name.clone(), + Expr::col((abilities::Entity, abilities::Column::Delegation)) + .equals((lookup_name.clone(), id.clone())), + ) + .and_where(within_budget()); + query.union(UnionType::All, abilities_query); + let mut revocations = snapshot_branch( + 3, + [ + node(lookup_name.clone()), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + null(), + ], + ); + revocations + .from(revocation::Entity) + .join( + JoinType::InnerJoin, + lookup_name, + Expr::col((revocation::Entity, revocation::Column::Revoked)) + .equals((Alias::new("lookup_nodes"), id)), + ) + .and_where(within_budget()); + query.union(UnionType::All, revocations); + query.with( + WithClause::new() + .recursive(true) + .cte(roots_cte) + .cte(edges_cte) + .cte(bounded_cte) + .cte(nodes_cte) + .cte(lookup_cte) + .to_owned(), + ) +} + +// The old metadata query had no ORDER BY. Authorization accepts any matching +// ability, but the last rejected caveat supplies a diagnostic reason. Use the +// typed `(resource, ability)` order on every backend; only that diagnostic can +// differ from the old backend-dependent order. +fn sort_abilities(abilities: &mut HashMap>) { + for rows in abilities.values_mut() { + rows.sort_unstable_by(|left, right| { + left.resource + .cmp(&right.resource) + .then_with(|| left.ability.cmp(&right.ability)) + }); + } +} +#[derive(Debug, PartialEq, Eq)] pub(crate) struct AuthGraphSnapshot { parents: HashMap>, delegations: HashMap, @@ -142,44 +456,93 @@ impl AuthGraphSnapshot { db: &C, roots: &[Hash], ) -> Result { - let (nodes, parents) = load_closure_edges(db, roots).await?; + let mut nodes = roots.to_vec(); + nodes.sort_by(|left, right| left.as_ref().cmp(right.as_ref())); + nodes.dedup(); + if nodes.len() > MAX_CHAIN_TRAVERSAL_NODES { + return Err(ChainTraversalError::LimitExceeded); + } if nodes.is_empty() { return Ok(Self { - parents, + parents: HashMap::new(), delegations: HashMap::new(), abilities: HashMap::new(), revoked: HashSet::new(), }); } - let delegations = delegation::Entity::find() - .filter(delegation::Column::Id.is_in(nodes.iter().copied())) - .all(db) - .await? - .into_iter() - .map(|row| (row.id, row)) - .collect(); - let mut ability_rows: HashMap> = HashMap::new(); - for row in abilities::Entity::find() - .filter(abilities::Column::Delegation.is_in(nodes.iter().copied())) - .all(db) - .await? - { - ability_rows.entry(row.delegation).or_default().push(row); + let rows = db + .query_all(db.get_database_backend().build(&snapshot_query(&nodes))) + .await?; + let mut parents: HashMap> = HashMap::new(); + let mut delegations = HashMap::new(); + let mut abilities: HashMap> = HashMap::new(); + let mut revoked = HashSet::new(); + let mut edge_count = 0; + for row in rows { + let node: Hash = row.try_get("", "node")?; + match row.try_get::("", "kind")? { + 0 => { + edge_count += 1; + if edge_count > MAX_CHAIN_TRAVERSAL_NODES * MAX_CHAIN_TRAVERSAL_NODES { + return Err(ChainTraversalError::LimitExceeded); + } + parents + .entry(node) + .or_default() + .push(row.try_get("", "parent")?); + } + 1 => { + delegations.insert( + node, + delegation::Model { + id: node, + delegator: row.try_get("", "delegator")?, + delegatee: row.try_get("", "delegatee")?, + expiry: row.try_get("", "expiry")?, + issued_at: row.try_get("", "issued_at")?, + not_before: row.try_get("", "not_before")?, + facts: row.try_get("", "facts")?, + serialization: row.try_get("", "serialization")?, + }, + ); + } + 2 => { + abilities.entry(node).or_default().push(abilities::Model { + delegation: node, + resource: row.try_get("", "resource")?, + ability: row.try_get("", "ability")?, + caveats: row.try_get("", "caveats")?, + }); + } + 3 => { + revoked.insert(node); + } + _ => return Err(DbErr::Type("unknown authorization graph row".into()).into()), + } + } + for ancestor_ids in parents.values_mut() { + ancestor_ids.sort_by(|left, right| left.as_ref().cmp(right.as_ref())); + ancestor_ids.dedup(); + } + sort_abilities(&mut abilities); + let mut visited = nodes.iter().copied().collect::>(); + let mut frontier = nodes.clone(); + while let Some(current) = frontier.pop() { + for parent in parents.get(¤t).map(Vec::as_slice).unwrap_or(&[]) { + if visited.insert(*parent) { + if visited.len() > MAX_CHAIN_TRAVERSAL_NODES { + return Err(ChainTraversalError::LimitExceeded); + } + nodes.push(*parent); + frontier.push(*parent); + } + } } - - let revoked = revocation::Entity::find() - .filter(revocation::Column::Revoked.is_in(nodes.iter().copied())) - .all(db) - .await? - .into_iter() - .map(|row| row.revoked) - .collect(); - Ok(Self { parents, delegations, - abilities: ability_rows, + abilities, revoked, }) } @@ -282,9 +645,10 @@ mod tests { use crate::hash::hash; use crate::migrations::Migrator; use crate::models::actor; + use crate::types::{Caveats, Facts}; use sea_orm::{ ActiveModelTrait, ActiveValue::Set, ConnectOptions, Database, DatabaseConnection, - QueryFilter, + DbBackend, EntityTrait, QueryFilter, Statement, }; use sea_orm_migration::MigratorTrait; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -310,6 +674,368 @@ mod tests { (db, counter) } + // Frozen pre-batch implementation used only to compare complete snapshots. + async fn load_sequential( + db: &C, + roots: &[Hash], + ) -> Result { + let (nodes, parents) = load_closure_edges(db, roots).await?; + if nodes.is_empty() { + return Ok(AuthGraphSnapshot { + parents, + delegations: HashMap::new(), + abilities: HashMap::new(), + revoked: HashSet::new(), + }); + } + let delegations = delegation::Entity::find() + .filter(delegation::Column::Id.is_in(nodes.iter().copied())) + .all(db) + .await? + .into_iter() + .map(|row| (row.id, row)) + .collect(); + let mut ability_rows: HashMap> = HashMap::new(); + for row in abilities::Entity::find() + .filter(abilities::Column::Delegation.is_in(nodes.iter().copied())) + .all(db) + .await? + { + ability_rows.entry(row.delegation).or_default().push(row); + } + // Compare the old loader's rows under the new explicit order, rather + // than accidentally pinning one backend's unspecified row order. + sort_abilities(&mut ability_rows); + let revoked = revocation::Entity::find() + .filter(revocation::Column::Revoked.is_in(nodes.iter().copied())) + .all(db) + .await? + .into_iter() + .map(|row| row.revoked) + .collect(); + Ok(AuthGraphSnapshot { + parents, + delegations, + abilities: ability_rows, + revoked, + }) + } + + #[tokio::test] + async fn snapshot_load_is_one_round_trip_and_matches_sequential() { + let (db, counter) = counted_database().await; + let ids = insert_chain(&db, "one-trip", 2).await; + let expected = load_sequential(&db, &[ids[0]]).await.unwrap(); + let before = counter.load(Ordering::SeqCst); + let actual = AuthGraphSnapshot::load(&db, &[ids[0]]).await.unwrap(); + assert_eq!(actual, expected); + assert_eq!(counter.load(Ordering::SeqCst) - before, 1); + } + + async fn assert_fixture_equivalence(db: &DatabaseConnection) { + actor::ActiveModel { + id: Set("did:key:actor".to_string()), + } + .insert(db) + .await + .unwrap(); + actor::ActiveModel { + id: Set("did:key:alias".to_string()), + } + .insert(db) + .await + .unwrap(); + let ids = insert_chain(db, "cross-backend", 3).await; + let alias = hash(b"cross-backend-alias"); + insert_delegation(db, alias).await; + insert_edge(db, alias, ids[2]).await; + insert_edge(db, ids[0], ids[2]).await; + + let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap(); + let mut windowed: delegation::ActiveModel = delegation::Entity::find_by_id(ids[1]) + .one(db) + .await + .unwrap() + .unwrap() + .into(); + windowed.delegator = Set("did:key:alias".to_string()); + windowed.issued_at = Set(Some(now - time::Duration::hours(1))); + windowed.not_before = Set(Some(now - time::Duration::minutes(15))); + windowed.expiry = Set(Some(now + time::Duration::hours(1))); + windowed.facts = Set(Some(Facts( + [( + "alias".to_string(), + serde_json::json!({"principal": "did:key:alias"}), + )] + .into(), + ))); + windowed.update(db).await.unwrap(); + + // Deliberately insert away from canonical `(resource, ability)` order. + // A one-ability fixture cannot detect changed rejection-reason order. + for id in [ids[0], ids[2], alias] { + for (index, action) in ["put", "get", "list", "del"].iter().enumerate() { + for suffix in ["z", "b", "a"] { + abilities::ActiveModel { + delegation: Set(id), + resource: Set(format!("tinycloud:did:key:actor:files/kv/{suffix}") + .parse() + .unwrap()), + ability: Set(format!("tinycloud.kv/{action}").try_into().unwrap()), + caveats: Set(Caveats( + [("limit".to_string(), serde_json::json!({"max": index + 1}))].into(), + )), + } + .insert(db) + .await + .unwrap(); + } + } + } + revocation::ActiveModel { + id: Set(hash(b"cross-backend-revocation")), + revoker: Set("did:key:actor".to_string()), + revoked: Set(ids[2]), + serialization: Set(b"revoked".to_vec()), + revoked_at: Set(Some(now)), + } + .insert(db) + .await + .unwrap(); + + for roots in [ + vec![ids[0]], + vec![alias, ids[0], alias], + vec![hash(b"missing-proof"), ids[1]], + ] { + let before = load_sequential(db, &roots).await.unwrap(); + let after = AuthGraphSnapshot::load(db, &roots).await.unwrap(); + assert_eq!(after, before, "roots {roots:?}"); + } + assert_eq!( + AuthGraphSnapshot::load(db, &[]).await.unwrap(), + load_sequential(db, &[]).await.unwrap() + ); + } + + async fn assert_canonical_caveat_reason(db: &DatabaseConnection) { + let id = hash(b"canonical-overlapping-caveats"); + delegation::ActiveModel { + id: Set(id), + delegator: Set("did:key:actor".to_string()), + delegatee: Set("did:key:alias".to_string()), + expiry: Set(None), + issued_at: Set(None), + not_before: Set(None), + facts: Set(None), + serialization: Set(id.as_ref().to_vec()), + } + .insert(db) + .await + .unwrap(); + let sql_caveats = Caveats( + [( + "0".to_string(), + serde_json::json!({ + "mode": "constrained-statements", + "readOnly": true, + "statements": [ + {"name": "get", "sql": "SELECT 1", "fixedParams": []} + ] + }), + )] + .into(), + ); + for (resource, caveats) in [ + ( + "tinycloud:did:key:actor:files/kv", + Caveats([("tenant".to_string(), serde_json::json!("a"))].into()), + ), + ("tinycloud:did:key:actor:files/kv/doc", sql_caveats.clone()), + ] { + abilities::ActiveModel { + delegation: Set(id), + resource: Set(resource.parse().unwrap()), + ability: Set("tinycloud.kv/get".to_string().try_into().unwrap()), + caveats: Set(caveats), + } + .insert(db) + .await + .unwrap(); + } + let snapshot = AuthGraphSnapshot::load(db, &[id]).await.unwrap(); + let rows = snapshot.abilities(&id); + assert_eq!(rows.len(), 2); + assert!( + rows[0].resource < rows[1].resource, + "canonical resource order" + ); + + let capability = crate::util::Capability { + resource: "tinycloud:did:key:actor:files/kv/doc".parse().unwrap(), + ability: "tinycloud.kv/get".to_string().try_into().unwrap(), + caveats: Caveats::default(), + }; + let parent = id.to_cid(0x55); + let err = crate::models::invocation::authorize_holder( + db, + "did:key:alias", + std::slice::from_ref(&capability), + &[parent], + OffsetDateTime::now_utc(), + ) + .await + .unwrap_err(); + assert!( + matches!( + &err, + crate::models::invocation::Error::InvalidInvocation( + crate::models::invocation::InvocationError::CaveatsNotContained(reason) + ) if reason == "containment-caveat-required" + ), + "unexpected canonical rejection: {err:?}" + ); + + // A matching constrained caveat still authorizes; sorting changes + // only which diagnostic wins if *all* candidates reject. + crate::models::invocation::authorize_holder( + db, + "did:key:alias", + &[crate::util::Capability { + caveats: sql_caveats, + ..capability + }], + &[parent], + OffsetDateTime::now_utc(), + ) + .await + .unwrap(); + } + + async fn assert_closure_limit(db: &DatabaseConnection) { + for size in [MAX_CHAIN_TRAVERSAL_NODES + 1, MAX_CHAIN_TRAVERSAL_NODES + 2] { + let ids = insert_chain(db, &format!("too-deep-{size}"), size - 1).await; + assert!(matches!( + AuthGraphSnapshot::load(db, &[ids[0]]).await, + Err(ChainTraversalError::LimitExceeded) + )); + } + + let ids = insert_chain(db, "boundary", MAX_CHAIN_TRAVERSAL_NODES - 1).await; + let snapshot = AuthGraphSnapshot::load(db, &[ids[0]]).await.unwrap(); + assert_eq!( + snapshot.chain_ids_from(&ids[0]).len(), + MAX_CHAIN_TRAVERSAL_NODES + ); + assert!( + snapshot.delegation(ids.last().unwrap()).is_some(), + "the 64th node must not be silently truncated" + ); + assert_eq!(snapshot, load_sequential(db, &[ids[0]]).await.unwrap()); + } + + #[tokio::test] + async fn sqlite_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + assert_fixture_equivalence(&db).await; + assert_canonical_caveat_reason(&db).await; + assert_closure_limit(&db).await; + } + + #[tokio::test] + async fn postgres_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases", + ) + .await + else { + return; + }; + assert_fixture_equivalence(&fixture.conn).await; + assert_canonical_caveat_reason(&fixture.conn).await; + assert_closure_limit(&fixture.conn).await; + } + + struct MySqlGraphFixture { + url: String, + name: String, + } + + impl Drop for MySqlGraphFixture { + fn drop(&mut self) { + let (url, name) = (self.url.clone(), self.name.clone()); + let result = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + Database::connect(url) + .await? + .execute(Statement::from_string( + DbBackend::MySql, + format!("DROP DATABASE IF EXISTS {name}"), + )) + .await + .map(|_| ()) + }) + }) + .join(); + if !matches!(result, Ok(Ok(()))) { + eprintln!("failed to drop isolated MySQL graph database {}", self.name); + } + } + } + + #[tokio::test] + async fn mysql_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { + let Ok(url) = std::env::var("TINYCLOUD_TEST_MYSQL_URL") else { + eprintln!("skipping MySQL graph snapshot equivalence (TINYCLOUD_TEST_MYSQL_URL unset)"); + return; + }; + let (prefix, _) = url + .rsplit_once('/') + .expect("MySQL URL must include a database"); + let name = format!( + "tc_graph_{}_{}", + std::process::id(), + OffsetDateTime::now_utc().unix_timestamp_nanos() + ); + let admin = Database::connect(url.clone()).await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::MySql, + format!("CREATE DATABASE {name}"), + )) + .await + .unwrap(); + let _fixture = MySqlGraphFixture { + url: url.clone(), + name: name.clone(), + }; + let db = Database::connect(format!("{prefix}/{name}")).await.unwrap(); + // The full repository migrator predates MySQL's indexed-column length + // limits. Keep this fixture scoped to the four auth-graph model tables + // rather than changing unrelated migrations for a query equivalence test. + for ddl in [ + "CREATE TABLE actor (id VARCHAR(191) PRIMARY KEY)", + "CREATE TABLE delegation (id VARBINARY(34) PRIMARY KEY, delegator VARCHAR(191) NOT NULL, delegatee VARCHAR(191) NOT NULL, expiry DATETIME NULL, issued_at DATETIME NULL, not_before DATETIME NULL, facts JSON NULL, serialization BLOB NOT NULL)", + "CREATE TABLE parent_delegation (child VARBINARY(34) NOT NULL, parent VARBINARY(34) NOT NULL, PRIMARY KEY (parent, child))", + "CREATE TABLE ability (resource VARCHAR(255) NOT NULL, ability VARCHAR(255) NOT NULL, delegation VARBINARY(34) NOT NULL, caveats JSON NOT NULL, PRIMARY KEY (resource, ability, delegation))", + "CREATE TABLE revocation (id VARBINARY(34) PRIMARY KEY, revoker VARCHAR(191) NOT NULL, revoked VARBINARY(34) NOT NULL, serialization BLOB NOT NULL, revoked_at DATETIME NULL)", + ] { + db.execute(Statement::from_string(DbBackend::MySql, ddl.to_string())) + .await + .unwrap(); + } + assert_fixture_equivalence(&db).await; + assert_canonical_caveat_reason(&db).await; + assert_closure_limit(&db).await; + db.close().await.unwrap(); + admin.close().await.unwrap(); + } + async fn insert_delegation(db: &DatabaseConnection, id: Hash) { delegation::ActiveModel { id: Set(id), @@ -371,23 +1097,6 @@ mod tests { } } - #[tokio::test] - async fn snapshot_load_fails_closed_at_traversal_limit() { - let (db, _) = counted_database().await; - let ids = insert_chain(&db, "deep", MAX_CHAIN_TRAVERSAL_NODES).await; - assert!(matches!( - AuthGraphSnapshot::load(&db, &[ids[0]]).await, - Err(ChainTraversalError::LimitExceeded) - )); - - let boundary = insert_chain(&db, "boundary", MAX_CHAIN_TRAVERSAL_NODES - 1).await; - let snapshot = AuthGraphSnapshot::load(&db, &[boundary[0]]).await.unwrap(); - assert_eq!( - snapshot.chain_ids_from(&boundary[0]).len(), - MAX_CHAIN_TRAVERSAL_NODES - ); - } - #[tokio::test] async fn snapshot_reports_direct_revocation() { let (db, _) = counted_database().await; @@ -464,9 +1173,7 @@ mod tests { /// depth-amplified number of queries (per-node closure walks repeated for /// locking, revocation, and chain-window checks, plus a revocation lookup /// per ancestor). The new path uses one pre-transaction closure query to - /// establish the lock set, then one transaction-scoped snapshot consisting - /// of one closure query and one batch each for delegations, abilities, and - /// revocations. + /// establish the lock set, then one tagged transaction-scoped snapshot query. #[tokio::test] async fn snapshot_query_batches_are_bounded_versus_depth_amplified_traversal() { for (depth, expected_legacy) in [(0, 6), (1, 10), (4, 22)] { @@ -502,7 +1209,7 @@ mod tests { let legacy_queries = counter.load(Ordering::SeqCst) - before; // New production path: one recursive edge query to establish the - // lock set, then a four-query transaction-scoped snapshot. + // lock set, then one transaction-scoped snapshot query. let before = counter.load(Ordering::SeqCst); let (lock_keys, _) = load_closure_edges(&db, &[leaf]).await.unwrap(); let snapshot = AuthGraphSnapshot::load(&db, &[leaf]).await.unwrap(); @@ -511,7 +1218,7 @@ mod tests { assert_eq!(lock_keys.len(), depth + 1); assert_eq!(snapshot.chain_ids_from(&leaf).len(), depth + 1); assert_eq!(legacy_queries, expected_legacy, "depth {depth}"); - assert_eq!(optimized_queries, 5, "depth {depth}"); + assert_eq!(optimized_queries, 2, "depth {depth}"); } } } diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index defe7e0e..c21ae8e9 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -161,6 +161,7 @@ mod meeting_snapshot_fence_tests { #[derive(Debug, Clone)] pub struct SpaceDatabase { conn: C, + candidate_conn: RecoveringConnection, storage: B, secrets: S, encryption: Option, @@ -319,6 +320,38 @@ where } } +/// Reject unauthorized or unhosted mutations before any content upload. +/// The commit transaction repeats authorization after the sequence lock. +async fn authorize_mutation_preupload( + conn: &C, + invocation: &Invocation, + mode: InvokeMode, + now: OffsetDateTime, + spaces: &[SpaceId], +) -> Result<(), TxError> { + match mode { + InvokeMode::Admitted => { + invocation::authorize_admitted(conn, &invocation.0, now, None) + .await + .map_err(TxError::::from)?; + } + InvokeMode::Public => { + invocation::verify_and_authorize(conn, &invocation.0, now) + .await + .map_err(TxError::::from)?; + } + InvokeMode::Internal | InvokeMode::MeetingPublication => {} + } + let hosted = space::Entity::find() + .filter(space::Column::Id.is_in(spaces.iter().cloned().map(SpaceIdWrap))) + .count(conn) + .await?; + if hosted != spaces.len() as u64 { + return Err(TxError::SpaceNotFound); + } + Ok(()) +} + /// Apply the schema migrations on `pool`. /// /// On PostgreSQL sea-orm-migration runs them in one transaction; it runs @@ -350,8 +383,9 @@ where migrate(&pool).await?; let writer_lock = (pool.get_database_backend() == sea_orm::DatabaseBackend::Sqlite) .then(|| Arc::new(tokio::sync::Mutex::new(()))); - let read_audit = ReadAuditPipeline::start(pool, writer_lock.clone()); + let read_audit = ReadAuditPipeline::start(pool.clone(), writer_lock.clone()); Ok(Self { + candidate_conn: pool, conn, storage, secrets, @@ -490,7 +524,11 @@ impl SpaceDatabase where C: TransactionTrait, { - /// Change the legacy artifact barrier. Freeze commit is the drain acknowledgement. + /// Change the legacy artifact barrier. Freeze commit acknowledges that + /// in-flight protected writes have committed or rolled back in the DB and + /// no protected upload can still land afterward. Cleanup of an unreferenced, + /// hash-fenced blob from a rolled-back write may finish after the ACK; it + /// only deletes/refunds physical storage, never changes committed KV rows. pub async fn set_legacy_meeting_write_freeze( &self, space: &SpaceId, @@ -500,16 +538,27 @@ where where C: ConnectionTrait, { - let _writer = match &self.writer_lock { - Some(lock) => Some(lock.lock().await), - None => None, - }; - let tx = self.begin_graph_transaction().await?; - let status = - crate::meeting_legacy_guard::transition(&tx, space, frozen, expected_generation) - .await?; - tx.commit().await?; - Ok(status) + // Acknowledgement must never wait indefinitely for a stalled upload. + // On timeout the transaction is dropped (and rolled back), so the + // caller cannot treat a failed freeze as a drained write barrier. + tokio::time::timeout(std::time::Duration::from_secs(30), async { + let _writer = match &self.writer_lock { + Some(lock) => Some(lock.lock().await), + None => None, + }; + let tx = self.begin_graph_transaction().await?; + let status = + crate::meeting_legacy_guard::transition(&tx, space, frozen, expected_generation) + .await?; + tx.commit().await?; + Ok(status) + }) + .await + .unwrap_or_else(|_| { + Err(crate::meeting_legacy_guard::FreezeError::Db(DbErr::Custom( + "legacy meeting freeze timed out waiting for in-flight writes".into(), + ))) + }) } pub async fn legacy_meeting_freeze_status( @@ -2086,62 +2135,138 @@ where } } + let mut legacy_spaces: Vec<_> = mutation_keys + .iter() + .filter(|(_, path)| crate::meeting_legacy_guard::protects(path.as_str())) + .map(|(space, _)| space) + .collect(); + legacy_spaces.sort_unstable(); + legacy_spaces.dedup(); + // Preserve frozen-write rejection before authorization and candidate + // registration. This read is only a preflight; the transaction takes + // the durable row lock before any upload below. + for space in &legacy_spaces { + if crate::meeting_legacy_guard::status(&self.conn, space) + .await + .map_err(|error| TxStoreError::Tx(TxError::Db(error)))? + .frozen + { + return Err(TxStoreError::LegacyMeetingFrozen); + } + } // Check the entire invocation before creating an object in the block // store. The transaction below repeats authorization after acquiring // the sequence lock, preserving revocation and time ordering. let now = OffsetDateTime::now_utc(); - match mode { - InvokeMode::Admitted => { - invocation::authorize_admitted(&self.conn, &invocation.0, now, None) - .await - .map_err(TxError::::from)?; - } - InvokeMode::Public => { - invocation::verify_and_authorize(&self.conn, &invocation.0, now) - .await - .map_err(TxError::::from)?; - } - InvokeMode::Internal | InvokeMode::MeetingPublication => {} - } let mut sequence_spaces = invocation.0.spaces().cloned().collect::>(); sequence_spaces.sort_unstable(); sequence_spaces.dedup(); - let hosted = space::Entity::find() - .filter(space::Column::Id.is_in(sequence_spaces.iter().cloned().map(SpaceIdWrap))) - .count(&self.conn) - .await?; - if hosted != sequence_spaces.len() as u64 { - return Err(TxError::SpaceNotFound.into()); - } let mut blobs = write_hashes .iter() .map(|((space, _), hash)| (space.clone(), *hash)) .collect::>(); blobs.sort_unstable(); blobs.dedup(); - if !blobs.is_empty() { - crate::kv_orphans::register_candidates(&self.conn, &blobs).await?; + if self.conn.get_database_backend() == sea_orm::DatabaseBackend::Postgres { + // Batch authorization, hosted-space check, and durable candidate + // registration on one connection. This preparatory transaction + // cannot commit a KV write or upload; replay it once after a lost + // connection even if its candidate COMMIT reached PostgreSQL. + let mut retried = false; + loop { + let tx = self.candidate_conn.begin_preparatory().await?; + let prepared: Result<(), TxError> = async { + authorize_mutation_preupload::<_, B, K>( + &tx, + &invocation, + mode, + now, + &sequence_spaces, + ) + .await?; + if !blobs.is_empty() { + crate::kv_orphans::register_candidates(&tx, &blobs).await?; + } + tx.commit().await?; + Ok(()) + } + .await; + match prepared { + Ok(()) => break, + Err(TxError::Db(error)) + if !retried && crate::db_pool::is_connection_error(&error) => + { + retried = true; + } + Err(error) => return Err(error.into()), + } + } + } else { + authorize_mutation_preupload::<_, B, K>( + &self.conn, + &invocation, + mode, + now, + &sequence_spaces, + ) + .await?; + if !blobs.is_empty() { + crate::kv_orphans::register_candidates_idempotent(&self.candidate_conn, &blobs) + .await?; + } } // The database transaction fences the blob hash against a collector // (and another writer of the same content) through commit or rollback. // PostgreSQL only takes the per-space advisory sequence lock *after* // all S3 uploads. SQLite has one writer transaction already. - let result = async { + // Keep the write-only transaction future off caller/test thread stacks: + // its auth graph and commit state grew beyond the default debug stack. + // Read-only invocations return above without this allocation. + let result = Box::pin(async { let _writer = match &self.writer_lock { Some(lock) => Some(lock.lock().await), None => None, }; - let begin_start = Instant::now(); - let tx_result = self.begin_graph_transaction().await; - crate::telemetry::observe_stage( - crate::telemetry::InvocationStage::DbTxBegin, - crate::telemetry::StageOutcome::from(tx_result.is_ok()), - begin_start.elapsed(), - ); - let tx = tx_result?; - if !blobs.is_empty() { - crate::kv_orphans::lock_candidates(&tx, &blobs).await?; - } + // A lost connection before persist cannot have landed a blob or + // committed a KV row. Restart the whole transaction once rather + // than replaying an individual statement on an aborted tx. This + // retakes both the hash fence and every protected freeze row. + let mut retried_preupload = false; + let tx = loop { + let begin_start = Instant::now(); + let tx_result = self.begin_graph_transaction().await; + crate::telemetry::observe_stage( + crate::telemetry::InvocationStage::DbTxBegin, + crate::telemetry::StageOutcome::from(tx_result.is_ok()), + begin_start.elapsed(), + ); + let tx = tx_result?; + let locked: Result = async { + if !blobs.is_empty() { + crate::kv_orphans::lock_candidates(&tx, &blobs).await?; + } + // A freeze waits for this row lock through commit/abort; + // a late writer is refused without uploading. + for space in &legacy_spaces { + if crate::meeting_legacy_guard::lock_writer(&tx, space).await? { + return Ok(true); + } + } + Ok(false) + } + .await; + match locked { + Ok(false) => break tx, + Ok(true) => return Err(TxStoreError::LegacyMeetingFrozen), + Err(error) + if !retried_preupload && crate::db_pool::is_connection_error(&error) => + { + retried_preupload = true; + drop(tx); + } + Err(error) => return Err(error.into()), + } + }; for ((space, path), stage) in stages.drain() { let hash = self .storage @@ -2159,18 +2284,6 @@ where // SQLite already holds its single-writer lock at BEGIN, preventing a // read-to-write upgrade race. Multi-space PG locks are acquired in // stable order to avoid cross-space deadlocks. - let mut legacy_spaces: Vec<_> = mutation_keys - .iter() - .filter(|(_, path)| crate::meeting_legacy_guard::protects(path.as_str())) - .map(|(space, _)| space) - .collect(); - legacy_spaces.sort_by_key(|space| space.to_string()); - legacy_spaces.dedup(); - for space in legacy_spaces { - if crate::meeting_legacy_guard::lock_writer(&tx, space).await? { - return Err(TxStoreError::LegacyMeetingFrozen); - } - } // DbTxBody spans post-begin to pre-commit. The guard defaults to an // `error` outcome so any `?`/early return inside the transaction is // recorded as a failure; it is disarmed to `ok` right before commit. @@ -2365,7 +2478,7 @@ where // commit tx if all side effects worked tx.commit().await?; Ok((commit, results)) - } + }) .await; if result.is_err() && !blobs.is_empty() { // The failed transaction (including any hash locks) has ended. @@ -4938,8 +5051,12 @@ mod test { db.invoke::(invocation, inputs).await }; assert!( - matches!(result, Err(TxStoreError::LegacyMeetingFrozen)), - "{ability} admitted={admitted}" + matches!(&result, Err(TxStoreError::LegacyMeetingFrozen)), + "{ability} admitted={admitted}: {:?}", + result + .as_ref() + .map(|_| ()) + .map_err(|error| error.to_string()) ); } } @@ -5089,6 +5206,11 @@ mod test { inner: MemoryStore, entered: Arc, release: Arc, + fail_after_persist: bool, + uploads: Arc, + pause_remove: bool, + removing: Arc, + remove_release: Arc, } #[async_trait::async_trait] impl StorageSetup for LegacyPausedStore { @@ -5128,9 +5250,15 @@ mod test { space: &SpaceId, stage: HashBuffer>, ) -> Result { + self.uploads + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); self.entered.notify_one(); self.release.notified().await; - self.inner.persist(space, stage).await + let hash = self.inner.persist(space, stage).await?; + if self.fail_after_persist { + return Err(std::io::Error::other("lost upload response")); + } + Ok(hash) } } #[async_trait::async_trait] @@ -5138,6 +5266,10 @@ mod test { type Error = std::io::Error; async fn remove(&self, space: &SpaceId, hash: &Hash) -> Result, Self::Error> { + if self.pause_remove { + self.removing.notify_one(); + self.remove_release.notified().await; + } self.inner.remove(space, hash).await } } @@ -5244,6 +5376,420 @@ mod test { )); } + #[tokio::test] + async fn postgres_legacy_freeze_during_slow_upload_waits_for_commit() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + let Some(schema) = crate::test_support::PostgresTestSchema::new( + "postgres_legacy_freeze_during_slow_upload_waits_for_commit", + ) + .await + else { + return; + }; + let storage = LegacyPausedStore::default(); + let first = SpaceDatabase::new( + schema.conn.clone(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let second = SpaceDatabase::new( + schema.conn.clone(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("pg-legacy-slow-upload"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&first.conn) + .await + .unwrap(); + let path: Path = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/slow" + .parse() + .unwrap(); + let writer_space = space.clone(); + let writer_path = path.clone(); + let writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"slow original").await.unwrap(); + first + .invoke_internal_kv_put::( + writer_space, + writer_path, + Metadata(Default::default()), + stage, + None, + ) + .await + .map_err(|error| error.to_string()) + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.entered.notified(), + ) + .await + .unwrap(); + let freeze_space = space.clone(); + let reader = second.clone(); + let mut freeze = tokio::spawn(async move { + second + .set_legacy_meeting_write_freeze(&freeze_space, true, 0) + .await + }); + let early_ack = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut freeze).await; + storage.release.notify_one(); + let written = tokio::time::timeout(std::time::Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap(); + let acknowledged_while_uploading = early_ack.is_ok(); + let frozen = match early_ack { + Ok(result) => result.unwrap().unwrap().frozen, + Err(_) => { + tokio::time::timeout(std::time::Duration::from_secs(10), freeze) + .await + .unwrap() + .unwrap() + .unwrap() + .frozen + } + }; + assert!( + !acknowledged_while_uploading, + "freeze acknowledged before pre-persisted write committed" + ); + let hash = written.unwrap(); + assert!(frozen); + assert!(storage.inner.contains(&space, &hash).await.unwrap()); + assert_eq!(reader.kv_get(&space, &path).await.unwrap().unwrap().1, hash); + } + + #[tokio::test] + async fn legacy_freeze_during_failed_upload_refunds_and_rejects_later_write_before_upload() { + use crate::storage::{memory::MemoryStaging, StoreSize}; + use futures::io::AsyncWriteExt; + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory.path().join("native.sqlite").display() + ); + let storage = LegacyPausedStore { + fail_after_persist: true, + ..Default::default() + }; + let first = SpaceDatabase::new( + Database::connect(url.clone()).await.unwrap(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let second = SpaceDatabase::new( + Database::connect(url).await.unwrap(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("legacy-failed-upload-freeze"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&first.conn) + .await + .unwrap(); + let path: Path = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/failed" + .parse() + .unwrap(); + let writer_space = space.clone(); + let writer_path = path.clone(); + let writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"failed upload").await.unwrap(); + first + .invoke_internal_kv_put::( + writer_space, + writer_path, + Metadata(Default::default()), + stage, + None, + ) + .await + .map_err(|error| error.to_string()) + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.entered.notified(), + ) + .await + .unwrap(); + let freeze_space = space.clone(); + let freeze_db = second.clone(); + let mut freeze = tokio::spawn(async move { + freeze_db + .set_legacy_meeting_write_freeze(&freeze_space, true, 0) + .await + }); + let early_ack = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut freeze).await; + storage.release.notify_one(); + let written = tokio::time::timeout(std::time::Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap(); + let acknowledged_while_uploading = early_ack.is_ok(); + let frozen = match early_ack { + Ok(result) => result.unwrap().unwrap().frozen, + Err(_) => { + tokio::time::timeout(std::time::Duration::from_secs(10), freeze) + .await + .unwrap() + .unwrap() + .unwrap() + .frozen + } + }; + assert!( + !acknowledged_while_uploading, + "freeze acknowledged while upload remained in flight" + ); + let error = written.unwrap_err(); + assert!(error.contains("lost upload response"), "{error}"); + assert!(frozen); + let hash = crate::hash::hash(b"failed upload"); + assert!(!storage.inner.contains(&space, &hash).await.unwrap()); + assert_eq!(storage.inner.total_size(&space).await.unwrap(), Some(0)); + assert_eq!( + crate::kv_orphans::collect_candidates(&second.conn, &storage, 1, time::Duration::ZERO) + .await + .unwrap(), + 0, + "immediate refund consumed candidate exactly once" + ); + let mut late_stage = HashBuffer::new(Vec::new()); + late_stage.write_all(b"late").await.unwrap(); + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(10), + second.invoke_internal_kv_put::( + space.clone(), + path, + Metadata(Default::default()), + late_stage, + None, + ) + ) + .await + .unwrap(), + Err(TxStoreError::LegacyMeetingFrozen) + )); + assert_eq!( + storage.uploads.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a post-freeze write must be refused before storage.persist" + ); + assert_eq!(storage.inner.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn postgres_legacy_freeze_ack_during_orphan_cleanup_preserves_committed_references() { + use crate::storage::{memory::MemoryStaging, StoreSize}; + use futures::io::AsyncWriteExt; + let Some(schema) = crate::test_support::PostgresTestSchema::new( + "postgres_legacy_freeze_ack_during_orphan_cleanup_preserves_committed_references", + ) + .await + else { + return; + }; + let storage = LegacyPausedStore { + fail_after_persist: true, + pause_remove: true, + ..Default::default() + }; + let committed_db = SpaceDatabase::new( + schema.conn.clone(), + storage.inner.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let writer_db = SpaceDatabase::new( + schema.conn.clone(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let freezer = SpaceDatabase::new( + schema.conn.clone(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("pg-freeze-orphan-cleanup"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&schema.conn) + .await + .unwrap(); + let kept_path: Path = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/kept" + .parse() + .unwrap(); + let mut kept_stage = HashBuffer::new(Vec::new()); + kept_stage.write_all(b"committed").await.unwrap(); + let kept_hash = committed_db + .invoke_internal_kv_put::( + space.clone(), + kept_path.clone(), + Metadata(Default::default()), + kept_stage, + None, + ) + .await + .map_err(|error| error.to_string()) + .unwrap(); + let before = get_kv_entity(&schema.conn, &space, &kept_path) + .await + .unwrap() + .unwrap(); + let writes_before = crate::models::kv_write::Entity::find() + .filter(crate::models::kv_write::Column::Space.eq(SpaceIdWrap(space.clone()))) + .count(&schema.conn) + .await + .unwrap(); + let orphan_path: Path = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/orphan" + .parse() + .unwrap(); + let orphan_probe = orphan_path.clone(); + let orphan_hash = crate::hash::hash(b"uncommitted"); + let writer_space = space.clone(); + let mut writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"uncommitted").await.unwrap(); + writer_db + .invoke_internal_kv_put::( + writer_space, + orphan_path, + Metadata(Default::default()), + stage, + None, + ) + .await + .map_err(|error| error.to_string()) + }); + if tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.entered.notified(), + ) + .await + .is_err() + { + writer.abort(); + panic!("the protected writer did not reach the upload"); + } + let freeze_space = space.clone(); + let mut freeze = tokio::spawn(async move { + freezer + .set_legacy_meeting_write_freeze(&freeze_space, true, 0) + .await + }); + let acknowledged_during_upload = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut freeze) + .await + .is_ok(); + storage.release.notify_one(); + if tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.removing.notified(), + ) + .await + .is_err() + { + storage.remove_release.notify_one(); + writer.abort(); + freeze.abort(); + panic!("the failed upload did not reach orphan cleanup"); + } + let ack_before_cleanup = + tokio::time::timeout(std::time::Duration::from_secs(10), &mut freeze).await; + let orphan_still_present = storage.inner.contains(&space, &orphan_hash).await; + storage.remove_release.notify_one(); + let orphan_still_present = orphan_still_present.unwrap(); + let write_result = + match tokio::time::timeout(std::time::Duration::from_secs(10), &mut writer).await { + Ok(result) => result.unwrap(), + Err(_) => { + writer.abort(); + freeze.abort(); + panic!("the failed write did not finish cleanup"); + } + }; + let acknowledged_before_cleanup = ack_before_cleanup.is_ok(); + let frozen = match ack_before_cleanup { + Ok(result) => result.unwrap().unwrap().frozen, + Err(_) => { + tokio::time::timeout(std::time::Duration::from_secs(10), freeze) + .await + .unwrap() + .unwrap() + .unwrap() + .frozen + } + }; + assert!(!acknowledged_during_upload); + assert!( + acknowledged_before_cleanup, + "the DB abort must release the freeze barrier" + ); + assert!( + orphan_still_present, + "cleanup was still pending at freeze ACK" + ); + assert!(frozen); + assert!(write_result.unwrap_err().contains("lost upload response")); + let after = get_kv_entity(&schema.conn, &space, &kept_path) + .await + .unwrap() + .unwrap(); + assert_eq!((after.seq, after.value), (before.seq, before.value)); + assert_eq!(after.value, kept_hash); + assert!(get_kv_entity(&schema.conn, &space, &orphan_probe) + .await + .unwrap() + .is_none()); + assert_eq!( + crate::models::kv_write::Entity::find() + .filter(crate::models::kv_write::Column::Space.eq(SpaceIdWrap(space.clone()))) + .count(&schema.conn) + .await + .unwrap(), + writes_before, + "post-ACK cleanup must not mutate committed rows" + ); + assert!(storage.inner.contains(&space, &kept_hash).await.unwrap()); + assert!(!storage.inner.contains(&space, &orphan_hash).await.unwrap()); + assert_eq!( + storage.inner.total_size(&space).await.unwrap(), + Some(b"committed".len() as u64), + "only the unreferenced candidate may be refunded" + ); + assert_eq!( + crate::kv_orphans::collect_candidates(&schema.conn, &storage, 1, time::Duration::ZERO) + .await + .unwrap(), + 0 + ); + } + #[tokio::test] async fn publication_cleanup_accepts_already_absent_legacy_keys() { use sea_orm::ActiveValue::Set; diff --git a/tinycloud-core/src/db_pool.rs b/tinycloud-core/src/db_pool.rs index 127be2aa..eeae6405 100644 --- a/tinycloud-core/src/db_pool.rs +++ b/tinycloud-core/src/db_pool.rs @@ -237,6 +237,35 @@ impl RecoveringConnection { /// the replay guard's `INSERT ... ON CONFLICT DO NOTHING` reports a /// conflict as a replay, so it must not be retried. pub async fn execute_idempotent(&self, stmt: Statement) -> Result { + self.execute_idempotent_with_tracking(stmt, true).await + } + + /// A replayable candidate reservation made before any user-visible KV + /// effect. Losing its connection cannot make the KV outcome unknown, even + /// if the reservation landed. Only the pre-upload orphan registry uses it. + pub(crate) async fn execute_idempotent_preparatory( + &self, + stmt: Statement, + ) -> Result { + self.execute_idempotent_with_tracking(stmt, false).await + } + + /// Pre-upload authorization and orphan registration share one connection. + /// This transaction can only change retryable candidate metadata; the + /// caller commits it before persisting a blob, so its uncertain outcome + /// cannot mean a user-visible KV write committed. + pub(crate) async fn begin_preparatory(&self) -> Result { + let transaction = self + .run("begin_preparatory", true, || self.conn.begin()) + .await?; + self.scope_local_timeout(transaction).await + } + + async fn execute_idempotent_with_tracking( + &self, + stmt: Statement, + track_user_write: bool, + ) -> Result { let started = Instant::now(); let result = match self.conn.execute(stmt.clone()).await { Err(error) if is_connection_error(&error) && !self.within_retry_window(started) => { @@ -252,7 +281,9 @@ impl RecoveringConnection { } result => result, }; - note_write_outcome(&result); + if track_user_write { + note_write_outcome(&result); + } result } diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index 7cec7935..91e98102 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -5,11 +5,12 @@ use std::{borrow::Cow, error::Error as StdError}; use sea_orm::{ sea_query::{Expr, LockBehavior, LockType, OnConflict}, ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, QueryFilter, - QueryOrder, QuerySelect, TransactionTrait, + QueryOrder, QuerySelect, QueryTrait, TransactionTrait, }; use time::{Duration, OffsetDateTime}; use tinycloud_auth::resource::SpaceId; +use crate::db_pool::RecoveringConnection; use crate::hash::Hash; use crate::models::{current_kv, kv_orphan_candidate as candidate, kv_write}; use crate::storage::ImmutableDeleteStore; @@ -53,6 +54,31 @@ pub async fn register_candidates( Ok(()) } +/// Register before upload on the recovering pool. Replaying the *same* +/// upsert after a lost connection is safe even if its first attempt committed: +/// the key and `created_at` are fixed in the statement. Transactional fences +/// use `register_candidates` instead and are never replayed independently. +pub async fn register_candidates_idempotent( + conn: &RecoveringConnection, + blobs: &[(SpaceId, Hash)], +) -> Result<(), DbErr> { + for (space, value) in ordered_unique(blobs).iter() { + let statement = candidate::Entity::insert(candidate::ActiveModel { + space: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), + value: sea_orm::ActiveValue::Set(*value), + created_at: sea_orm::ActiveValue::Set(OffsetDateTime::now_utc()), + }) + .on_conflict( + OnConflict::columns([candidate::Column::Space, candidate::Column::Value]) + .update_column(candidate::Column::CreatedAt) + .to_owned(), + ) + .build(conn.get_database_backend()); + conn.execute_idempotent_preparatory(statement).await?; + } + Ok(()) +} + /// Fence each blob against concurrent collection before uploading it. /// /// Call after registration, inside the KV transaction, before any PUT or diff --git a/tinycloud-node-server/tests/postgres_connection_recovery.rs b/tinycloud-node-server/tests/postgres_connection_recovery.rs index 845b142b..ca203fa9 100644 --- a/tinycloud-node-server/tests/postgres_connection_recovery.rs +++ b/tinycloud-node-server/tests/postgres_connection_recovery.rs @@ -804,6 +804,124 @@ async fn postgres_requests_survive_one_stale_pooled_connection() { fixture.drop_schema().await; } +/// Candidate registration is an idempotent write before upload. A stale idle +/// connection must be retried here, before any blob can land in storage. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn postgres_preupload_candidate_registration_retries_stale_connection() { + let Some(url) = + postgres_url("postgres_preupload_candidate_registration_retries_stale_connection") + else { + return; + }; + let fixture = Fixture::new("candidate", &url, None).await; + fixture.fill_pool().await; + let victim = fixture.order_idle_queue(0).await; + fixture.terminate(&BTreeSet::from([victim])).await; + let before = fixture.stats(); + let blob = ( + fixture.owner.space.clone(), + tinycloud_core::hash::hash(b"preupload-stale-connection"), + ); + tinycloud_core::kv_orphans::register_candidates_idempotent(&fixture.conn, &[blob]) + .await + .expect("candidate is registered after reconnect"); + let recovered = delta(before, fixture.stats()); + assert_eq!( + ( + recovered.losses, + recovered.retries, + recovered.idempotent_retries + ), + (1, 1, 1), + "{recovered:?}" + ); + assert_eq!(fixture.row_count("kv_orphan_candidate").await, 1); + fixture.drop_schema().await; +} + +/// A protected write crosses the durable freeze row before storage.persist. +/// Losing an idle pooled connection must not bypass that fence or strand its +/// candidate; the resulting KV remains readable after the recovered commit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn postgres_protected_preupload_fence_survives_stale_connection() { + let Some(url) = postgres_url("postgres_protected_preupload_fence_survives_stale_connection") + else { + return; + }; + let mut fixture = Fixture::new("protected", &url, None).await; + fixture.fill_pool().await; + let victim = fixture.order_idle_queue(0).await; + fixture.terminate(&BTreeSet::from([victim])).await; + let before = fixture.stats(); + let key = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/stale-pool"; + fixture + .put(key) + .await + .expect("protected put survives reconnect"); + fixture.get(key).await.expect("committed blob is readable"); + let recovered = delta(before, fixture.stats()); + assert_eq!( + (recovered.losses, recovered.retries), + (1, 1), + "{recovered:?}" + ); + assert_eq!(fixture.row_count("kv_orphan_candidate").await, 0); + fixture.drop_schema().await; +} + +/// A dead backend during the protected row lock rolls back that transaction. +/// Because storage has not run yet, the invocation can acquire a fresh +/// transaction, retake the fence, then upload and commit exactly once. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn postgres_preupload_freeze_lock_recovers_a_lost_transaction() { + let Some(url) = postgres_url("postgres_preupload_freeze_lock_recovers_a_lost_transaction") + else { + return; + }; + let mut fixture = Fixture::new("guardloss", &url, None).await; + fixture + .admin + .execute_unprepared(&format!( + "CREATE SEQUENCE {schema}.kill_once; \ + CREATE FUNCTION {schema}.kill_guard_once() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN \ + IF nextval('{schema}.kill_once') = 1 THEN \ + PERFORM pg_terminate_backend(pg_backend_pid()); \ + END IF; \ + RETURN NEW; \ + END $$; \ + CREATE TRIGGER kill_guard_once BEFORE INSERT ON {schema}.meeting_legacy_write_guard \ + FOR EACH ROW EXECUTE FUNCTION {schema}.kill_guard_once()", + schema = fixture.schema + )) + .await + .unwrap(); + let key = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/guard-loss"; + fixture + .put(key) + .await + .expect("retry preupload transaction after dead guard connection"); + fixture + .get(key) + .await + .expect("only committed value survives"); + let killed = fixture + .admin + .query_one(Statement::from_string( + DbBackend::Postgres, + format!("SELECT last_value AS n FROM {}.kill_once", fixture.schema), + )) + .await + .unwrap() + .unwrap() + .try_get::("", "n") + .unwrap(); + assert_eq!(killed, 2, "row lock was retried exactly once"); + assert_eq!(fixture.row_count("kv_write").await, 2); // seed plus this put + assert_eq!(fixture.row_count("kv_orphan_candidate").await, 0); + fixture.drop_schema().await; +} + /// A network cut: a proxy resets every client connection while idle (EOF/RST, /// no FATAL message). Requests still succeed. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/tinycloud-node-server/tests/postgres_round_trip_budget.rs b/tinycloud-node-server/tests/postgres_round_trip_budget.rs index 95e33b00..6c469472 100644 --- a/tinycloud-node-server/tests/postgres_round_trip_budget.rs +++ b/tinycloud-node-server/tests/postgres_round_trip_budget.rs @@ -113,15 +113,24 @@ use tokio::{ net::{TcpListener, TcpStream}, }; -/// Round trips allowed per flow; see the module docs. Measured with -/// `test_before_acquire(false)`, database identity resolve as one statement, -/// the read audit as one statement and no second sequence lock (TC-951): +/// Measured warm on PostgreSQL with #302's one-query authorization snapshot +/// (required dependency of TC-735). Sign-in and GET each lose six trips: +/// - sign-in 69 = peer key 0 + host `/delegate` 17 + session `/delegate` 18 +/// + account-space SQL write 34; +/// - kv get 16, including its durable read-audit commit. /// -/// - sign-in 75 = peer key 0 + host `/delegate` 17 + session `/delegate` 18 -/// + account-space SQL write 40; -/// - kv put 37; -/// - kv get 22, including the read-audit commit. -const BUDGETS: [(&str, u64); 3] = [("sign-in", 75), ("kv put", 37), ("kv get", 22)]; +/// KV PUT is 42 = 35 statements + seven mandatory pool-release pings. +/// Against the pre-#302 production 37 = 31 statements + six pings, the four +/// net new statements are seven necessary steps minus three queries saved by +/// #302 in the commit authorization graph: BEGIN/COMMIT pin pre-upload work +/// for retry-safe atomicity (two), the pre-upload authorization graph and +/// hosted-space COUNT reject before S3 (two), the candidate reservation +/// survives a crash before upload (one), a second hash-fence reservation +/// under the commit transaction excludes GC after upload (one), and the +/// candidate DELETE clears the committed reference atomically (one). +/// The extra preflight transaction also releases one pooled connection, +/// whose ping flushes cancelled/unfinished statements and cannot be removed. +const BUDGETS: [(&str, u64); 3] = [("sign-in", 69), ("kv put", 42), ("kv get", 16)]; /// Full sign-in plus put plus get rounds run before the measured one. const WARM_UP_ROUNDS: u8 = 3; From ac1994dc6c246e7e617471864488b9493fd03c0c Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 00:30:12 +0000 Subject: [PATCH 05/17] TC-735: fence orphan cleanup with blob generations --- CHANGELOG.md | 4 +- docs/kv-sync.md | 90 +- tinycloud-core/src/db.rs | 686 ++++++- tinycloud-core/src/db_pool.rs | 44 +- tinycloud-core/src/events/mod.rs | 5 + tinycloud-core/src/kv_orphans.rs | 1780 ++++++++++++++--- tinycloud-core/src/kv_sync.rs | 139 +- .../migrations/m20260724_010000_current_kv.rs | 11 +- .../m20260725_000000_request_path_indexes.rs | 2 + .../m20261015_020000_kv_orphan_candidates.rs | 58 +- tinycloud-core/src/models/current_kv.rs | 2 + tinycloud-core/src/models/invocation.rs | 6 + .../src/models/kv_blob_generation.rs | 18 + .../src/models/kv_orphan_attempt.rs | 22 + .../src/models/kv_orphan_candidate.rs | 3 + tinycloud-core/src/models/kv_write.rs | 2 + tinycloud-core/src/models/mod.rs | 2 + tinycloud-core/src/storage/either.rs | 101 +- tinycloud-core/src/storage/memory.rs | 80 +- tinycloud-core/src/storage/mod.rs | 58 +- tinycloud-node-server/src/lib.rs | 13 +- tinycloud-node-server/src/routes/mod.rs | 11 + .../src/storage/file_system.rs | 206 +- tinycloud-node-server/src/storage/mod.rs | 7 + tinycloud-node-server/src/storage/s3.rs | 641 +++++- .../tests/postgres_connection_recovery.rs | 32 +- .../tests/postgres_kv_write_throughput.rs | 54 +- .../tests/postgres_round_trip_budget.rs | 23 +- 28 files changed, 3456 insertions(+), 644 deletions(-) create mode 100644 tinycloud-core/src/models/kv_blob_generation.rs create mode 100644 tinycloud-core/src/models/kv_orphan_attempt.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f883b780..7f3e158e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## [Unreleased] -- KV puts persist content-addressed blobs before taking the per-space sequence lock, while the lock still orders committed `kv/sync` cursors. Protected legacy meeting writes acquire the durable freeze barrier before upload, so a freeze waits for their commit or abort and fails after 30 seconds if stalled. A durable hash-fenced candidate table (`m20261015_020000_kv_orphan_candidates`) supports immediate cleanup and physical-byte quota refunds after known failures; a 60-second collector reclaims crash leftovers after a five-minute grace. Quota remains conservative and per process, not a durable global ledger (TC-735). - Lost connections during candidate registration are retried idempotently; before any upload, a lost hash/freeze-lock transaction can restart once with both fences. Neither an upload nor a commit is replayed on an uncertain outcome. +- KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). + Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/docs/kv-sync.md b/docs/kv-sync.md index 06e24760..a4f5977d 100644 --- a/docs/kv-sync.md +++ b/docs/kv-sync.md @@ -132,41 +132,55 @@ Read audit records every poll. Poll on demand, or no more often than every ## Write ordering and blob recovery -KV puts stage and authorize content before upload. The node registers a -durable `(space, content hash)` recovery candidate, then persists the blob -before taking the per-space sequence lock. The database transaction rechecks -authorization and conditional-write preconditions under that lock; it holds -the lock through commit, so `seq` and sync cursors remain commit-ordered even -when same-space uploads finish out of order. An unauthorized write uploads -nothing. - -For protected legacy meeting paths, the durable freeze row is locked before -upload and held until commit or abort. That barrier is distinct from the -per-space sequence lock: a freeze drains uploads already in flight, while -writes arriving after the freeze are rejected before upload. A stalled writer -causes the freeze attempt to fail after 30 seconds rather than acknowledge an -incomplete drain. - -Before upload, a lost pooled connection retries the candidate registration -with the same idempotent upsert. If the hash or freeze row lock loses its -transaction, the node restarts that transaction once and retakes both fences -before persisting anything. It never replays an upload or committed write -after a connection loss. - -A failed upload, precondition, or database write triggers an immediate -hash-fenced reference check. Unreferenced blobs are removed and their -physical-byte quota is refunded before the failure returns. Historical -`kv_write` references are retained as well as current values, because a -client can still read an older value. The candidate row is removed with a -successful KV commit; a failed cleanup leaves it for the collector. After a -crash, the node scans candidates at 60-second intervals, collecting those -older than five minutes in bounded batches of 256 until the eligible backlog -is drained. S3 and filesystem stores reserve bytes before a potentially -ambiguous write; deleting the resulting object or canceling a reservation -for an object that never materialized refunds once, not twice. - -Quota is conservative physical accounting, **not commit-only accounting**: -in-flight uploads and crash leftovers may temporarily count against it. -`SpaceSizes` remains in-memory and per process; writes or deletions on other -nodes do not synchronize that counter. Globally exact multi-node accounting -requires a separate shared byte ledger (TC-1162). +KV puts stage and authorize content before upload. A short preparatory +transaction checks authorization and hosting, then commits an active +`(space, content hash, generation)` fence and a separate claim for each +writer. The first generation is zero and retains the original bare-hash +storage key; a generation above zero is allocated only after a prior +generation's physical deletion is confirmed. Ordinary writes release every +pooled connection and database lock before S3 or filesystem I/O. A +four-minute upload deadline stays below the five-minute DB-clock GC grace. +After upload, an ordinary writer takes the per-space sequence guard, starts +a fresh commit transaction and takes +the DB sequence lock; it verifies its own active claim, then rechecks +authorization and conditional-write preconditions. The guards stay through +commit, so `seq` and sync cursors remain commit-ordered even when same-space +uploads finish out of order. An unauthorized write uploads nothing. + +Protected legacy meeting paths are the TC-1260 exception: the durable freeze +row and its pooled transaction remain locked across upload until commit or +abort. A freeze acknowledgement drains earlier uploads; a later protected +write is refused before upload. A stalled writer causes the freeze attempt +to fail after 30 seconds rather than acknowledge an incomplete drain. + +Before upload, a lost pooled connection retries the preparatory transaction +with the same attempt ID. The registration is idempotent; neither an upload +nor a committed write is replayed on an uncertain outcome. A missing claim +or a fence already deleting returns a retryable error, never a reference to +a potentially removed object. + +A failed upload, precondition or database write removes only its own claim. +If another claim or a current or historical KV reference exists, it leaves +the shared generation's blob alone. Otherwise it commits a `deleting` fence +and byte entitlement, releases the DB connection, then deletes that +generation's object. A writer seeing `deleting` cannot deduplicate or +upload over it. Only confirmed physical absence refunds the per-process +quota exactly once; only then is the fence retired and the next generation +made available. Each delete targets its original generation's storage key, +so even a delayed second delete cannot remove a later committed generation. +A lost DELETE response or a crash mid-delete leaves the fence for +idempotent GC retry. Reads get the generation from their existing KV lookup, +without another database query; generation zero reads the bare key. A +rolled-back binary cannot read records with generation above zero, which +can occur after a failed upload is collected and its content is saved again. + +The collector runs every 60 seconds, aging attempts after five minutes by +DB time and draining eligible work in batches of 256. On S3 startup the +same object-listing snapshot reconstructs both `SpaceSizes` and which +`deleting` fences counted physical bytes; a deletion before the listing +needs no refund, while a deletion afterward refunds once. Orphans present +at restart count until GC. Quota is conservative physical accounting, +**not commit-only accounting**: pending uploads and crash leftovers can +temporarily over-count. `SpaceSizes` remains in-memory and per process; other +nodes' writes and deletes do not synchronize it. Globally exact multi-node +accounting requires a separate shared byte ledger (TC-1162). diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index c21ae8e9..4dac0ebc 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -21,6 +21,7 @@ use crate::types::{ Resource, SpaceIdWrap, }; use crate::util::{Capability, DelegationInfo, DelegationMode}; +use rand::{rngs::OsRng, RngCore}; use sea_orm::{ entity::prelude::*, error::{DbErr, RuntimeErr, SqlxError}, @@ -53,6 +54,10 @@ use tinycloud_auth::{ /// cap still does; the cap is what bounds HTTP/2 over TLS, whose header list /// may reach 16 MiB. pub const KV_MAX_MUTATIONS_PER_INVOCATION: usize = 4096; +/// All persist calls in one invocation finish before the five-minute orphan GC grace. +const KV_UPLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(240); +const _: () = + assert!(KV_UPLOAD_TIMEOUT.as_secs() < crate::kv_orphans::ORPHAN_GRACE.whole_seconds() as u64); pub const HOOK_DELIVERY_STATUS_PENDING: &str = "pending"; pub const HOOK_DELIVERY_STATUS_RETRYING: &str = "retrying"; @@ -169,6 +174,9 @@ pub struct SpaceDatabase { revocation_chain_locks: ChainLockRegistry, kv_object_locks: KvObjectLockRegistry, space_sequence_locks: SpaceSequenceLockRegistry, + // SQLite lock order everywhere: per-space sequence guard, then writer + // mutex, then BEGIN. Freeze takes only the writer mutex. PostgreSQL has no + // writer mutex and takes its sequence guard after uploading. writer_lock: Option>>, read_audit: ReadAuditPipeline, database_fence_cache: Arc, @@ -284,6 +292,8 @@ where MeetingSnapshotProtected, #[error("legacy meeting artifacts are frozen")] LegacyMeetingFrozen, + #[error("KV blob candidate is unavailable; retry the invocation")] + KvCandidateUnavailable, #[error("KV precondition failed")] KvPreconditionFailed, #[error("KV response is {size} bytes, exceeding the requested limit of {limit} bytes")] @@ -1257,15 +1267,21 @@ where range: ByteRangeSpec, ) -> Result)>, EitherError> { - let Some((metadata, hash)) = metadata_with_hash(&self.conn, space_id, key) + let Some(entry) = get_kv_entity(&self.conn, space_id, key) .await .map_err(EitherError::A)? else { return Ok(None); }; + let current_kv::Model { + metadata, + value: hash, + generation, + .. + } = entry; let Some(content) = self .storage - .read_range(space_id, &hash, range) + .read_range(space_id, &hash, generation, range) .await .map_err(EitherError::B)? else { @@ -2121,6 +2137,7 @@ where key: path.clone(), metadata, value, + generation: 0, }); } // add delete for tx @@ -2167,118 +2184,217 @@ where .collect::>(); blobs.sort_unstable(); blobs.dedup(); - if self.conn.get_database_backend() == sea_orm::DatabaseBackend::Postgres { - // Batch authorization, hosted-space check, and durable candidate - // registration on one connection. This preparatory transaction - // cannot commit a KV write or upload; replay it once after a lost - // connection even if its candidate COMMIT reached PostgreSQL. - let mut retried = false; - loop { + // GC ages attempts from registration, not from storage.persist. + // This deadline also covers preparatory retries and queueing behind + // legacy freeze locks, so no PUT may begin after its five-minute fence + // grace could expire. + let upload_deadline = tokio::time::Instant::now() + KV_UPLOAD_TIMEOUT; + let mut attempt_bytes = [0u8; 16]; + OsRng.fill_bytes(&mut attempt_bytes); + let attempt_id = hex::encode(attempt_bytes); + // Registration and authorization commit before any blob can be + // persisted. Retrying an uncertain preparatory COMMIT uses exactly the + // same attempt ID, so it cannot duplicate or steal another writer. + // SQLite's preparatory write follows sequence→writer→BEGIN and + // releases every guard before an ordinary upload. + let mut retried = false; + let blobs = loop { + let _sequence = if self.writer_lock.is_some() { + self.acquire_space_sequence_guards(&sequence_spaces).await + } else { + Vec::new() + }; + let _writer = match &self.writer_lock { + Some(lock) => Some(lock.lock().await), + None => None, + }; + let prepared = async { let tx = self.candidate_conn.begin_preparatory().await?; - let prepared: Result<(), TxError> = async { - authorize_mutation_preupload::<_, B, K>( - &tx, - &invocation, - mode, - now, - &sequence_spaces, - ) - .await?; - if !blobs.is_empty() { - crate::kv_orphans::register_candidates(&tx, &blobs).await?; - } - tx.commit().await?; - Ok(()) + authorize_mutation_preupload::<_, B, K>( + &tx, + &invocation, + mode, + now, + &sequence_spaces, + ) + .await?; + let generations = if blobs.is_empty() { + Vec::new() + } else { + crate::kv_orphans::register_candidates(&tx, &blobs, &attempt_id).await? + }; + tx.commit().await?; + Ok::<_, TxError>(generations) + } + .await; + match prepared { + Ok(generations) => break generations, + Err(TxError::Db(error)) + if !retried && crate::db_pool::is_connection_error(&error) => + { + retried = true; } - .await; - match prepared { - Ok(()) => break, - Err(TxError::Db(error)) - if !retried && crate::db_pool::is_connection_error(&error) => - { - retried = true; - } - Err(error) => return Err(error.into()), + Err(TxError::Db(error)) if crate::kv_orphans::is_candidate_unavailable(&error) => { + return Err(TxStoreError::KvCandidateUnavailable); } + Err(error) => return Err(error.into()), } - } else { - authorize_mutation_preupload::<_, B, K>( - &self.conn, - &invocation, - mode, - now, - &sequence_spaces, - ) - .await?; - if !blobs.is_empty() { - crate::kv_orphans::register_candidates_idempotent(&self.candidate_conn, &blobs) - .await?; + }; + for op in &mut ops { + if let Operation::KvWrite { + space, + value, + generation, + .. + } = op + { + let index = blobs + .binary_search_by(|(s, v, _)| (s, v).cmp(&(&*space, &*value))) + .expect("registered generation for every staged write"); + *generation = blobs[index].2; } } - // The database transaction fences the blob hash against a collector - // (and another writer of the same content) through commit or rollback. - // PostgreSQL only takes the per-space advisory sequence lock *after* - // all S3 uploads. SQLite has one writer transaction already. // Keep the write-only transaction future off caller/test thread stacks: // its auth graph and commit state grew beyond the default debug stack. // Read-only invocations return above without this allocation. let result = Box::pin(async { - let _writer = match &self.writer_lock { - Some(lock) => Some(lock.lock().await), - None => None, - }; - // A lost connection before persist cannot have landed a blob or - // committed a KV row. Restart the whole transaction once rather - // than replaying an individual statement on an aborted tx. This - // retakes both the hash fence and every protected freeze row. - let mut retried_preupload = false; - let tx = loop { - let begin_start = Instant::now(); - let tx_result = self.begin_graph_transaction().await; - crate::telemetry::observe_stage( - crate::telemetry::InvocationStage::DbTxBegin, - crate::telemetry::StageOutcome::from(tx_result.is_ok()), - begin_start.elapsed(), - ); - let tx = tx_result?; - let locked: Result = async { - if !blobs.is_empty() { - crate::kv_orphans::lock_candidates(&tx, &blobs).await?; - } - // A freeze waits for this row lock through commit/abort; - // a late writer is refused without uploading. - for space in &legacy_spaces { - if crate::meeting_legacy_guard::lock_writer(&tx, space).await? { - return Ok(true); + let begin_mutation = || { + let legacy_spaces = &legacy_spaces; + async move { + // Retrying a lost BEGIN or freeze lock is safe before + // any KV transaction body or COMMIT has run. + let mut retried = false; + loop { + let begin_start = Instant::now(); + let tx_result = self.begin_graph_transaction().await; + crate::telemetry::observe_stage( + crate::telemetry::InvocationStage::DbTxBegin, + crate::telemetry::StageOutcome::from(tx_result.is_ok()), + begin_start.elapsed(), + ); + let tx = match tx_result { + Ok(tx) => tx, + Err(error) + if !retried && crate::db_pool::is_connection_error(&error) => + { + retried = true; + continue; + } + Err(error) => return Err(TxStoreError::Tx(TxError::Db(error))), + }; + let locked: Result = async { + // Protected legacy writes take the durable freeze + // row before uploading and hold it through commit. + for space in legacy_spaces { + if crate::meeting_legacy_guard::lock_writer(&tx, space).await? { + return Ok(true); + } + } + Ok(false) + } + .await; + match locked { + Ok(false) => break Ok(tx), + Ok(true) => break Err(TxStoreError::LegacyMeetingFrozen), + Err(error) + if !retried && crate::db_pool::is_connection_error(&error) => + { + retried = true; + drop(tx); + } + Err(error) => break Err(error.into()), } } - Ok(false) } - .await; - match locked { - Ok(false) => break tx, - Ok(true) => return Err(TxStoreError::LegacyMeetingFrozen), - Err(error) - if !retried_preupload && crate::db_pool::is_connection_error(&error) => - { - retried_preupload = true; - drop(tx); + }; + // TC-1260: protected legacy paths deliberately retain the old + // lock_writer transaction across upload. Freeze ACK must wait + // until such an in-flight write commits or aborts. Ordinary paths + // never hold a transaction, pool connection, sequence or writer + // lock during upload. + let protected = !legacy_spaces.is_empty(); + let _legacy_sqlite_sequence = if protected && self.writer_lock.is_some() { + self.acquire_space_sequence_guards(&sequence_spaces).await + } else { + Vec::new() + }; + let _legacy_writer = if protected { + match &self.writer_lock { + Some(lock) => Some(lock.lock().await), + None => None, + } + } else { + None + }; + let legacy_tx = if protected { + Some(begin_mutation().await?) + } else { + None + }; + if !blobs.is_empty() { + if tokio::time::Instant::now() >= upload_deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "KV upload deadline elapsed before persist", + ) + .into()); + } + tokio::time::timeout_at(upload_deadline, async { + for (key, stage) in stages.drain() { + let expected = write_hashes[&key]; + let (space, _) = key; + let index = blobs + .binary_search_by(|(s, v, _)| (s, v).cmp(&(&space, &expected))) + .expect("registered generation for every staged write"); + let hash = self + .storage + .persist(&space, stage, blobs[index].2) + .await + .map_err(TxStoreError::StoreWrite)?; + debug_assert_eq!(hash, expected); } - Err(error) => return Err(error.into()), + Ok::<(), TxStoreError>(()) + }) + .await + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "KV upload timed out") + })??; + } + // PG sequence guards are taken after upload. SQLite takes them + // before its writer mutex and BEGIN. The protected SQLite writer + // already holds both guards above. + let _sequence_guards = if protected && self.writer_lock.is_some() { + Vec::new() + } else { + self.acquire_space_sequence_guards(&sequence_spaces).await + }; + let _ordinary_writer = if protected { + None + } else { + match &self.writer_lock { + Some(lock) => Some(lock.lock().await), + None => None, } }; - for ((space, path), stage) in stages.drain() { - let hash = self - .storage - .persist(&space, stage) + let tx = match legacy_tx { + Some(tx) => tx, + None => begin_mutation().await?, + }; + crate::kv_sync::lock_space_sequences(&tx, &sequence_spaces).await?; + // Legacy freeze alone holds its transaction across upload. Take + // the hash fence only now, after all external storage I/O, for + // both legacy and ordinary writes. + if !blobs.is_empty() { + crate::kv_orphans::lock_candidates(&tx, &blobs, &attempt_id) .await - .map_err(TxStoreError::StoreWrite)?; - debug_assert_eq!(write_hashes.get(&(space, path)), Some(&hash)); + .map_err(|error| { + if crate::kv_orphans::is_candidate_unavailable(&error) { + TxStoreError::KvCandidateUnavailable + } else { + error.into() + } + })?; } - // Only the database section assigns sequence numbers. Its advisory - // lock remains held through commit so kv/sync cursors stay ordered. - let _sequence_guards = self.acquire_space_sequence_guards(&sequence_spaces).await; - crate::kv_sync::lock_space_sequences(&tx, &sequence_spaces).await?; // The sequence lock precedes every precondition read under READ // COMMITTED, including when an upload was prepared concurrently. // SQLite already holds its single-writer lock at BEGIN, preventing a @@ -2468,7 +2584,7 @@ where }; } if !blobs.is_empty() { - crate::kv_orphans::finish_candidates(&tx, &blobs).await?; + crate::kv_orphans::finish_candidates(&tx, &blobs, &attempt_id).await?; } // Record the transaction body as successful (post-begin to @@ -2485,7 +2601,8 @@ where // A commit error may be outcome-unknown: the candidate fence and // committed-reference check distinguish that from an orphan. if let Err(error) = - crate::kv_orphans::refund_failed(&self.conn, &self.storage, &blobs).await + crate::kv_orphans::refund_failed(&self.conn, &self.storage, &blobs, &attempt_id) + .await { tracing::warn!( ?error, @@ -3945,7 +4062,7 @@ async fn get_kv( }; let content_hash = e.value; let c = match store - .read(space_id, &content_hash) + .read(space_id, &content_hash, e.generation) .await .map_err(EitherError::B)? { @@ -5017,6 +5134,242 @@ mod test { ) } + #[tokio::test] + async fn sqlite_prepersist_upload_does_not_invert_delegate_lock_order() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory.path().join("lock-order.db").display() + ); + let storage = LegacyPausedStore::default(); + let db = SpaceDatabase::new( + Database::connect(url).await.unwrap(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("sqlite-lock-order"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&db.conn) + .await + .unwrap(); + let writer_db = db.clone(); + let writer_space = space.clone(); + let writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"paused upload").await.unwrap(); + writer_db + .invoke_internal_kv_put::( + writer_space, + "lock/key".parse().unwrap(), + Metadata(Default::default()), + stage, + None, + ) + .await + .map_err(|error| error.to_string()) + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.entered.notified(), + ) + .await + .unwrap(); + let invocation = + legacy_test_invocation(&space, &"lock/other".parse().unwrap(), "tinycloud.kv/del"); + // Keep the generic transaction live after it takes the sequence + // guard. A one-shot poll would leave it suspended indefinitely and + // incorrectly report a deadlock even when both paths have one order. + let writer_gate = tokio::time::timeout( + std::time::Duration::from_secs(3), + db.writer_lock.as_ref().unwrap().lock(), + ) + .await + .expect("ordinary upload must not hold the SQLite writer mutex"); + let delegate_db = db.clone(); + let delegate = tokio::spawn(async move { + delegate_db + .transact(vec![Event::InternalInvocation( + Box::new(invocation), + vec![], + )]) + .await + .map_err(|error| error.to_string()) + }); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + let acquired = db + .space_sequence_locks + .lock() + .await + .get(&space) + .and_then(Weak::upgrade) + .is_some_and(|lock| lock.try_lock().is_err()); + if acquired { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("generic transaction must acquire the sequence before writer"); + drop(writer_gate); + storage.release.notify_one(); + let (write, delegated) = tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::join!(writer, delegate) + }) + .await + .expect("KV upload and generic transaction must not deadlock"); + assert!(write.unwrap().is_ok()); + assert!(delegated.unwrap().is_ok()); + } + + #[tokio::test] + async fn sqlite_ordinary_upload_does_not_hold_writer_or_sequence() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory.path().join("ordinary.db").display() + ); + let storage = LegacyPausedStore::default(); + let db = SpaceDatabase::new( + Database::connect(url).await.unwrap(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("sqlite-ordinary-pool-release"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&db.conn) + .await + .unwrap(); + let writer_db = db.clone(); + let writer_space = space.clone(); + let writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"ordinary upload").await.unwrap(); + writer_db + .invoke_internal_kv_put::( + writer_space, + "notes/paused".parse().unwrap(), + Metadata(Default::default()), + stage, + None, + ) + .await + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.entered.notified(), + ) + .await + .unwrap(); + // GC can run while an ordinary S3/FS upload is in flight. The + // attempt is newer than its five-minute grace, so it stays durable + // until commit even though the blob has not landed yet. + let attempts_before = crate::models::kv_orphan_attempt::Entity::find() + .all(&db.conn) + .await + .unwrap(); + assert_eq!(attempts_before.len(), 1); + assert_eq!( + crate::kv_orphans::collect_candidates_with_progress( + &db.conn, + &storage, + 256, + crate::kv_orphans::ORPHAN_GRACE, + ) + .await + .unwrap(), + (0, 0) + ); + assert_eq!( + crate::models::kv_orphan_attempt::Entity::find() + .all(&db.conn) + .await + .unwrap(), + attempts_before + ); + let invocation = legacy_test_invocation( + &space, + &"notes/deleted".parse().unwrap(), + "tinycloud.kv/del", + ); + tokio::time::timeout( + std::time::Duration::from_secs(3), + db.transact(vec![Event::InternalInvocation( + Box::new(invocation), + vec![], + )]), + ) + .await + .expect("SQLite writer and sequence must be free during ordinary upload") + .map_err(|error| error.to_string()) + .unwrap(); + storage.release.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .map_err(|error| error.to_string()) + .unwrap(); + } + + #[tokio::test] + async fn unauthorized_ordinary_put_never_reaches_storage() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory.path().join("unauthorized.db").display() + ); + let storage = LegacyPausedStore::default(); + let db = SpaceDatabase::new( + Database::connect(url).await.unwrap(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("unauthorized-ordinary"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&db.conn) + .await + .unwrap(); + let path: Path = "notes/unauthorized".parse().unwrap(); + // The signed invocation presents an unregistered delegation. + let invocation = legacy_test_invocation(&space, &path, "tinycloud.kv/put"); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"unauthorized").await.unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(3), + db.invoke::( + invocation, + HashMap::from([((space, path), (Metadata(Default::default()), stage))]), + ), + ) + .await + .expect("authorization should fail before persist"); + assert!(result.is_err()); + assert_eq!(storage.uploads.load(std::sync::atomic::Ordering::SeqCst), 0,); + } + #[tokio::test] async fn legacy_freeze_covers_public_admitted_put_delete_and_deprecated_delete() { use crate::storage::memory::MemoryStaging; @@ -5223,23 +5576,30 @@ mod test { impl ImmutableReadStore for LegacyPausedStore { type Error = std::io::Error; type Readable = futures::io::Cursor>; - async fn contains(&self, space: &SpaceId, hash: &Hash) -> Result { - self.inner.contains(space, hash).await + async fn contains( + &self, + space: &SpaceId, + hash: &Hash, + generation: i64, + ) -> Result { + self.inner.contains(space, hash, generation).await } async fn read( &self, space: &SpaceId, hash: &Hash, + generation: i64, ) -> Result>, Self::Error> { - self.inner.read(space, hash).await + self.inner.read(space, hash, generation).await } async fn read_range( &self, space: &SpaceId, hash: &Hash, + generation: i64, range: crate::storage::ByteRangeSpec, ) -> Result>, Self::Error> { - self.inner.read_range(space, hash, range).await + self.inner.read_range(space, hash, generation, range).await } } #[async_trait::async_trait] @@ -5249,12 +5609,13 @@ mod test { &self, space: &SpaceId, stage: HashBuffer>, + generation: i64, ) -> Result { self.uploads .fetch_add(1, std::sync::atomic::Ordering::SeqCst); self.entered.notify_one(); self.release.notified().await; - let hash = self.inner.persist(space, stage).await?; + let hash = self.inner.persist(space, stage, generation).await?; if self.fail_after_persist { return Err(std::io::Error::other("lost upload response")); } @@ -5265,12 +5626,17 @@ mod test { impl ImmutableDeleteStore for LegacyPausedStore { type Error = std::io::Error; - async fn remove(&self, space: &SpaceId, hash: &Hash) -> Result, Self::Error> { + async fn remove( + &self, + space: &SpaceId, + hash: &Hash, + generation: i64, + ) -> Result, Self::Error> { if self.pause_remove { self.removing.notify_one(); self.remove_release.notified().await; } - self.inner.remove(space, hash).await + self.inner.remove(space, hash, generation).await } } #[tokio::test] @@ -5376,6 +5742,98 @@ mod test { )); } + #[tokio::test] + async fn postgres_ordinary_upload_releases_pool_and_sequence_before_persist() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + + let Some(schema) = crate::test_support::PostgresTestSchema::new( + "postgres_ordinary_upload_releases_pool_and_sequence_before_persist", + ) + .await + else { + return; + }; + let storage = LegacyPausedStore::default(); + let db = SpaceDatabase::new( + schema.conn.clone(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("pg-ordinary-pool-release"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&db.conn) + .await + .unwrap(); + let pool = schema.conn.get_postgres_connection_pool(); + // The fixture's max_connections is four: reserve three and leave + // precisely one for both preparatory and final transactions. + let mut held = Vec::new(); + for _ in 0..3 { + held.push(pool.acquire().await.unwrap()); + } + let baseline = pool.size() as usize - pool.num_idle(); + let writer_db = db.clone(); + let writer_space = space.clone(); + let writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"ordinary upload").await.unwrap(); + writer_db + .invoke_internal_kv_put::( + writer_space, + "notes/paused".parse().unwrap(), + Metadata(Default::default()), + stage, + None, + ) + .await + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + storage.entered.notified(), + ) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if pool.size() as usize - pool.num_idle() == baseline { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("ordinary upload must release its pooled connection while paused"); + // A different same-space transaction must commit while upload is paused. + let invocation = legacy_test_invocation( + &space, + &"notes/deleted".parse().unwrap(), + "tinycloud.kv/del", + ); + tokio::time::timeout( + std::time::Duration::from_secs(3), + db.transact(vec![Event::InternalInvocation( + Box::new(invocation), + vec![], + )]), + ) + .await + .expect("another same-space write should not wait for an ordinary upload") + .map_err(|error| error.to_string()) + .unwrap(); + storage.release.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .map_err(|error| error.to_string()) + .unwrap(); + } + #[tokio::test] async fn postgres_legacy_freeze_during_slow_upload_waits_for_commit() { use crate::storage::memory::MemoryStaging; @@ -5466,7 +5924,7 @@ mod test { ); let hash = written.unwrap(); assert!(frozen); - assert!(storage.inner.contains(&space, &hash).await.unwrap()); + assert!(storage.inner.contains(&space, &hash, 0).await.unwrap()); assert_eq!(reader.kv_get(&space, &path).await.unwrap().unwrap().1, hash); } @@ -5563,7 +6021,7 @@ mod test { assert!(error.contains("lost upload response"), "{error}"); assert!(frozen); let hash = crate::hash::hash(b"failed upload"); - assert!(!storage.inner.contains(&space, &hash).await.unwrap()); + assert!(!storage.inner.contains(&space, &hash, 0).await.unwrap()); assert_eq!(storage.inner.total_size(&space).await.unwrap(), Some(0)); assert_eq!( crate::kv_orphans::collect_candidates(&second.conn, &storage, 1, time::Duration::ZERO) @@ -5721,7 +6179,7 @@ mod test { } let ack_before_cleanup = tokio::time::timeout(std::time::Duration::from_secs(10), &mut freeze).await; - let orphan_still_present = storage.inner.contains(&space, &orphan_hash).await; + let orphan_still_present = storage.inner.contains(&space, &orphan_hash, 0).await; storage.remove_release.notify_one(); let orphan_still_present = orphan_still_present.unwrap(); let write_result = @@ -5775,8 +6233,12 @@ mod test { writes_before, "post-ACK cleanup must not mutate committed rows" ); - assert!(storage.inner.contains(&space, &kept_hash).await.unwrap()); - assert!(!storage.inner.contains(&space, &orphan_hash).await.unwrap()); + assert!(storage.inner.contains(&space, &kept_hash, 0).await.unwrap()); + assert!(!storage + .inner + .contains(&space, &orphan_hash, 0) + .await + .unwrap()); assert_eq!( storage.inner.total_size(&space).await.unwrap(), Some(b"committed".len() as u64), @@ -5932,6 +6394,7 @@ mod test { epoch: epoch_id, epoch_seq: 0, value: crate::hash::hash(b"shared-value"), + generation: 0, metadata: Metadata(std::collections::BTreeMap::new()), }; kv_write::ActiveModel::from(write.clone()) @@ -6303,6 +6766,7 @@ mod test { epoch: Set(crate::hash::hash(b"query-plan-epoch")), epoch_seq: Set(0), value: Set(crate::hash::hash(format!("value-{seq}").as_bytes())), + generation: Set(0), metadata: Set(Metadata(std::collections::BTreeMap::new())), }); } diff --git a/tinycloud-core/src/db_pool.rs b/tinycloud-core/src/db_pool.rs index eeae6405..ed654106 100644 --- a/tinycloud-core/src/db_pool.rs +++ b/tinycloud-core/src/db_pool.rs @@ -237,35 +237,6 @@ impl RecoveringConnection { /// the replay guard's `INSERT ... ON CONFLICT DO NOTHING` reports a /// conflict as a replay, so it must not be retried. pub async fn execute_idempotent(&self, stmt: Statement) -> Result { - self.execute_idempotent_with_tracking(stmt, true).await - } - - /// A replayable candidate reservation made before any user-visible KV - /// effect. Losing its connection cannot make the KV outcome unknown, even - /// if the reservation landed. Only the pre-upload orphan registry uses it. - pub(crate) async fn execute_idempotent_preparatory( - &self, - stmt: Statement, - ) -> Result { - self.execute_idempotent_with_tracking(stmt, false).await - } - - /// Pre-upload authorization and orphan registration share one connection. - /// This transaction can only change retryable candidate metadata; the - /// caller commits it before persisting a blob, so its uncertain outcome - /// cannot mean a user-visible KV write committed. - pub(crate) async fn begin_preparatory(&self) -> Result { - let transaction = self - .run("begin_preparatory", true, || self.conn.begin()) - .await?; - self.scope_local_timeout(transaction).await - } - - async fn execute_idempotent_with_tracking( - &self, - stmt: Statement, - track_user_write: bool, - ) -> Result { let started = Instant::now(); let result = match self.conn.execute(stmt.clone()).await { Err(error) if is_connection_error(&error) && !self.within_retry_window(started) => { @@ -281,12 +252,21 @@ impl RecoveringConnection { } result => result, }; - if track_user_write { - note_write_outcome(&result); - } + note_write_outcome(&result); result } + /// Pre-upload authorization and orphan registration share one connection. + /// This transaction can only change retryable candidate metadata; the + /// caller commits it before persisting a blob, so its uncertain outcome + /// cannot mean a user-visible KV write committed. + pub(crate) async fn begin_preparatory(&self) -> Result { + let transaction = self + .run("begin_preparatory", true, || self.conn.begin()) + .await?; + self.scope_local_timeout(transaction).await + } + /// The outcome of the one retry. It is never retried again, but a /// connection lost again is still recorded. fn retried(&self, operation: &'static str, result: Result) -> Result { diff --git a/tinycloud-core/src/events/mod.rs b/tinycloud-core/src/events/mod.rs index 8b3d99f1..32a92212 100644 --- a/tinycloud-core/src/events/mod.rs +++ b/tinycloud-core/src/events/mod.rs @@ -56,6 +56,7 @@ pub(crate) enum Operation { space: SpaceId, key: Path, value: Hash, + generation: i64, metadata: Metadata, }, KvDelete { @@ -72,11 +73,13 @@ impl Operation { space, key, value, + generation, metadata, } => VersionedOperation::KvWrite { space, key, value, + generation, metadata, seq, epoch, @@ -111,6 +114,7 @@ pub(crate) enum VersionedOperation { space: SpaceId, key: Path, value: Hash, + generation: i64, metadata: Metadata, seq: i64, epoch: Hash, @@ -224,6 +228,7 @@ fn hash_inv(inv_hash: &Hash, sp: &SpaceId, ops: &[Operation]) -> Result Some(Op::KvWrite { key, value: value.to_cid(CBOR_CODEC), diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index 91e98102..f7706a07 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -5,18 +5,29 @@ use std::{borrow::Cow, error::Error as StdError}; use sea_orm::{ sea_query::{Expr, LockBehavior, LockType, OnConflict}, ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, QueryFilter, - QueryOrder, QuerySelect, QueryTrait, TransactionTrait, + QueryOrder, QuerySelect, Statement, TransactionTrait, }; use time::{Duration, OffsetDateTime}; use tinycloud_auth::resource::SpaceId; -use crate::db_pool::RecoveringConnection; use crate::hash::Hash; -use crate::models::{current_kv, kv_orphan_candidate as candidate, kv_write}; -use crate::storage::ImmutableDeleteStore; +use crate::models::{ + current_kv, kv_blob_generation as ledger, kv_orphan_attempt as attempt, + kv_orphan_candidate as candidate, kv_write, +}; +use crate::storage::{ImmutableDeleteStore, ImmutableReadStore}; use crate::types::SpaceIdWrap; const MAX_BATCH_SIZE: u64 = 256; +/// Expired registrations are collected only after the upload deadline. +pub const ORPHAN_GRACE: Duration = Duration::minutes(5); + +const UNAVAILABLE_DELETING: &str = "KV blob is being deleted; retry registration"; +const UNAVAILABLE_MISSING: &str = "KV blob attempt lost its active fence"; + +pub(crate) fn is_candidate_unavailable(error: &DbErr) -> bool { + matches!(error, DbErr::Custom(message) if message == UNAVAILABLE_DELETING || message == UNAVAILABLE_MISSING) +} fn ordered_unique(blobs: &[(SpaceId, Hash)]) -> Cow<'_, [(SpaceId, Hash)]> { if blobs.windows(2).all(|pair| pair[0] < pair[1]) { @@ -28,176 +39,701 @@ fn ordered_unique(blobs: &[(SpaceId, Hash)]) -> Cow<'_, [(SpaceId, Hash)]> { Cow::Owned(ordered) } -/// Durably register every possible orphan before uploading any of these blobs. -/// -/// An existing candidate gets a new grace period. This operation is intentionally -/// outside the KV transaction so a failed precondition or a crash leaves a -/// collectable candidate behind. +/// Register this invocation's durable claim in a short preparatory transaction. +/// The caller commits before uploading. Replaying the same ID is harmless. pub async fn register_candidates( conn: &C, blobs: &[(SpaceId, Hash)], -) -> Result<(), DbErr> { - for (space, value) in ordered_unique(blobs).iter() { - candidate::Entity::insert(candidate::ActiveModel { - space: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), - value: sea_orm::ActiveValue::Set(*value), - created_at: sea_orm::ActiveValue::Set(OffsetDateTime::now_utc()), - }) - .on_conflict( - OnConflict::columns([candidate::Column::Space, candidate::Column::Value]) - .update_column(candidate::Column::CreatedAt) - .to_owned(), - ) - .exec(conn) - .await?; - } - Ok(()) -} - -/// Register before upload on the recovering pool. Replaying the *same* -/// upsert after a lost connection is safe even if its first attempt committed: -/// the key and `created_at` are fixed in the statement. Transactional fences -/// use `register_candidates` instead and are never replayed independently. -pub async fn register_candidates_idempotent( - conn: &RecoveringConnection, - blobs: &[(SpaceId, Hash)], -) -> Result<(), DbErr> { + attempt_id: &str, +) -> Result, DbErr> { + let mut registered_at = None; + let mut generations = Vec::with_capacity(blobs.len()); for (space, value) in ordered_unique(blobs).iter() { - let statement = candidate::Entity::insert(candidate::ActiveModel { - space: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), - value: sea_orm::ActiveValue::Set(*value), - created_at: sea_orm::ActiveValue::Set(OffsetDateTime::now_utc()), - }) - .on_conflict( - OnConflict::columns([candidate::Column::Space, candidate::Column::Value]) - .update_column(candidate::Column::CreatedAt) + let key = SpaceIdWrap(space.clone()); + if conn.get_database_backend() == DbBackend::Postgres { + // The ledger upsert serializes registration against deletion. A + // deleting fence returns no row, so no caller starts an upload. + let row = conn + .query_one(Statement::from_sql_and_values( + DbBackend::Postgres, + "WITH generation AS ( + INSERT INTO kv_blob_generation (space, value, generation) + VALUES ($1, $2, 0) + ON CONFLICT (space, value) DO UPDATE + SET generation = kv_blob_generation.generation + RETURNING space, value, generation + ), fence AS ( + INSERT INTO kv_orphan_candidate + (space, value, generation, created_at, deleting_at, deletion_bytes) + SELECT space, value, generation, CURRENT_TIMESTAMP, NULL, NULL + FROM generation WHERE true + ON CONFLICT (space, value) DO UPDATE + SET created_at = EXCLUDED.created_at + WHERE kv_orphan_candidate.deleting_at IS NULL + AND kv_orphan_candidate.generation = EXCLUDED.generation + RETURNING space, value, generation + ), claim AS ( + INSERT INTO kv_orphan_attempt (space, value, attempt_id, generation, created_at) + SELECT space, value, $3, generation, CURRENT_TIMESTAMP FROM fence WHERE true + ON CONFLICT (space, value, attempt_id) DO NOTHING + RETURNING attempt_id + ) + SELECT (SELECT generation FROM fence) AS generation, + (SELECT COUNT(*) FROM claim) AS inserted", + [key.into(), (*value).into(), attempt_id.to_owned().into()], + )) + .await? + .ok_or_else(|| DbErr::Custom("KV candidate registration returned no row".into()))?; + let generation: Option = row.try_get("", "generation")?; + generations.push(( + space.clone(), + *value, + generation.ok_or_else(|| DbErr::Custom(UNAVAILABLE_DELETING.into()))?, + )); + continue; + } + if conn.get_database_backend() == DbBackend::MySql { + conn.execute(Statement::from_sql_and_values( + DbBackend::MySql, + "INSERT INTO kv_blob_generation (space, value, generation) VALUES (?, ?, 0) + ON DUPLICATE KEY UPDATE generation = generation", + [key.clone().into(), (*value).into()], + )) + .await?; + } else { + let inserted = ledger::Entity::insert(ledger::ActiveModel { + space: sea_orm::ActiveValue::Set(key.clone()), + value: sea_orm::ActiveValue::Set(*value), + generation: sea_orm::ActiveValue::Set(0), + }) + .on_conflict( + OnConflict::columns([ledger::Column::Space, ledger::Column::Value]) + .do_nothing() + .to_owned(), + ) + .exec(conn) + .await; + match inserted { + Ok(_) | Err(DbErr::RecordNotInserted) => {} + Err(error) => return Err(error), + } + } + let generation = if conn.get_database_backend() == DbBackend::MySql { + ledger::Entity::find_by_id((key.clone(), *value)) + .lock_exclusive() + .one(conn) + .await? + } else { + ledger::Entity::find_by_id((key.clone(), *value)) + .one(conn) + .await? + } + .ok_or_else(|| DbErr::Custom("KV generation ledger missing".into()))? + .generation; + if conn.get_database_backend() == DbBackend::MySql { + conn.execute(Statement::from_sql_and_values( + DbBackend::MySql, + "INSERT INTO kv_orphan_candidate + (space, value, generation, created_at, deleting_at, deletion_bytes) + VALUES (?, ?, ?, UTC_TIMESTAMP(6), NULL, NULL) + ON DUPLICATE KEY UPDATE created_at = created_at", + [key.clone().into(), (*value).into(), generation.into()], + )) + .await?; + } else { + let inserted = candidate::Entity::insert(candidate::ActiveModel { + space: sea_orm::ActiveValue::Set(key.clone()), + value: sea_orm::ActiveValue::Set(*value), + generation: sea_orm::ActiveValue::Set(generation), + created_at: sea_orm::ActiveValue::Set(OffsetDateTime::now_utc()), + deleting_at: sea_orm::ActiveValue::Set(None), + deletion_bytes: sea_orm::ActiveValue::Set(None), + }) + .on_conflict( + OnConflict::columns([candidate::Column::Space, candidate::Column::Value]) + .do_nothing() + .to_owned(), + ) + .exec(conn) + .await; + match inserted { + Ok(_) | Err(DbErr::RecordNotInserted) => {} + Err(error) => return Err(error), + } + } + let now = match registered_at { + Some(now) => now, + None => { + let now = database_now(conn).await?; + registered_at = Some(now); + now + } + }; + let active = if conn.get_database_backend() == DbBackend::Sqlite { + candidate::Entity::update_many() + .col_expr(candidate::Column::CreatedAt, Expr::value(now)) + .filter(candidate::Column::Space.eq(key.clone())) + .filter(candidate::Column::Value.eq(*value)) + .filter(candidate::Column::Generation.eq(generation)) + .filter(candidate::Column::DeletingAt.is_null()) + .exec(conn) + .await? + .rows_affected + == 1 + } else { + let row = candidate::Entity::find_by_id((key.clone(), *value)) + .lock_exclusive() + .one(conn) + .await?; + if row.is_some_and(|row| row.deleting_at.is_none() && row.generation == generation) { + candidate::Entity::update_many() + .col_expr(candidate::Column::CreatedAt, Expr::value(now)) + .filter(candidate::Column::Space.eq(key.clone())) + .filter(candidate::Column::Value.eq(*value)) + .exec(conn) + .await?; + true + } else { + false + } + }; + if !active { + return Err(DbErr::Custom(UNAVAILABLE_DELETING.into())); + } + if conn.get_database_backend() == DbBackend::MySql { + conn.execute(Statement::from_sql_and_values( + DbBackend::MySql, + "INSERT INTO kv_orphan_attempt (space, value, attempt_id, generation, created_at) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE attempt_id = attempt_id", + [ + key.into(), + (*value).into(), + attempt_id.to_owned().into(), + generation.into(), + now.into(), + ], + )) + .await?; + } else { + let inserted = attempt::Entity::insert(attempt::ActiveModel { + space: sea_orm::ActiveValue::Set(key), + value: sea_orm::ActiveValue::Set(*value), + attempt_id: sea_orm::ActiveValue::Set(attempt_id.to_owned()), + generation: sea_orm::ActiveValue::Set(generation), + created_at: sea_orm::ActiveValue::Set(now), + }) + .on_conflict( + OnConflict::columns([ + attempt::Column::Space, + attempt::Column::Value, + attempt::Column::AttemptId, + ]) + .do_nothing() .to_owned(), - ) - .build(conn.get_database_backend()); - conn.execute_idempotent_preparatory(statement).await?; + ) + .exec(conn) + .await; + match inserted { + Ok(_) | Err(DbErr::RecordNotInserted) => {} + Err(error) => return Err(error), + } + } + generations.push((space.clone(), *value, generation)); } - Ok(()) + Ok(generations) } -/// Fence each blob against concurrent collection before uploading it. -/// -/// Call after registration, inside the KV transaction, before any PUT or -/// per-space sequence lock. Hold the transaction through the upload and -/// successful KV commit. Sorting prevents reversed hash-lock deadlocks. +/// Validate ownership inside the final KV transaction; never create a missing +/// fence here. The caller already finished ordinary storage I/O. pub async fn lock_candidates( tx: &DatabaseTransaction, - blobs: &[(SpaceId, Hash)], + blobs: &[(SpaceId, Hash, i64)], + attempt_id: &str, ) -> Result<(), DbErr> { - // Upsert again inside the transaction: a previous writer may have - // committed and deleted our pre-registered row while we waited to begin - // this transaction. The upsert owns the hash fence through commit on - // PostgreSQL, MySQL and SQLite, even when the row was absent. - register_candidates(tx, blobs).await + for (space, value, generation) in blobs { + let key = SpaceIdWrap(space.clone()); + if tx.get_database_backend() == DbBackend::Postgres { + // The joined row lock covers both the active fence and this exact + // durable attempt. GC removes attempts only after taking that + // same fence lock, so neither row can disappear before commit. + let owned = tx + .query_one(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT 1 AS owned FROM kv_orphan_candidate AS fence + JOIN kv_orphan_attempt AS claim + ON claim.space = fence.space AND claim.value = fence.value + AND claim.generation = fence.generation + WHERE fence.space = $1 AND fence.value = $2 + AND fence.generation = $4 + AND fence.deleting_at IS NULL AND claim.attempt_id = $3 + FOR UPDATE OF fence", + [ + key.into(), + (*value).into(), + attempt_id.to_owned().into(), + (*generation).into(), + ], + )) + .await? + .is_some(); + if !owned { + return Err(DbErr::Custom(UNAVAILABLE_MISSING.into())); + } + continue; + } + let active = if tx.get_database_backend() == DbBackend::Sqlite { + candidate::Entity::update_many() + .col_expr( + candidate::Column::CreatedAt, + Expr::col(candidate::Column::CreatedAt).into(), + ) + .filter(candidate::Column::Space.eq(key.clone())) + .filter(candidate::Column::Value.eq(*value)) + .filter(candidate::Column::Generation.eq(*generation)) + .filter(candidate::Column::DeletingAt.is_null()) + .exec(tx) + .await? + .rows_affected + == 1 + } else { + candidate::Entity::find_by_id((key.clone(), *value)) + .lock_exclusive() + .one(tx) + .await? + .is_some_and(|row| row.deleting_at.is_none() && row.generation == *generation) + }; + if !active { + return Err(DbErr::Custom(UNAVAILABLE_MISSING.into())); + } + let own = attempt::Entity::find_by_id((key, *value, attempt_id.to_owned())) + .filter(attempt::Column::Generation.eq(*generation)); + // MySQL repeatable-read may already have an older snapshot from KV + // authorization. A locking read observes attempts removed by GC. + let own = if tx.get_database_backend() == DbBackend::MySql { + own.lock_exclusive() + } else { + own + }; + if own.one(tx).await?.is_none() { + return Err(DbErr::Custom(UNAVAILABLE_MISSING.into())); + } + } + Ok(()) } -/// Remove successful candidates atomically with their KV writes. -/// -/// Call only once the writes have succeeded and before committing the KV -/// transaction. Rollback leaves candidates for the collector. +/// Remove only this invocation's claims in the same transaction as its KV refs. pub async fn finish_candidates( tx: &DatabaseTransaction, - blobs: &[(SpaceId, Hash)], + blobs: &[(SpaceId, Hash, i64)], + attempt_id: &str, ) -> Result<(), DbErr> { - for (space, value) in ordered_unique(blobs).iter() { - candidate::Entity::delete_by_id((SpaceIdWrap(space.clone()), *value)) + for (space, value, generation) in blobs { + let key = SpaceIdWrap(space.clone()); + if tx.get_database_backend() == DbBackend::Postgres { + // This is only the *successful* KV commit path: transact already + // wrote the reference in this transaction. Its earlier + // lock_candidates call holds this fence; `locked` also makes the + // dependency explicit for the single statement. PostgreSQL CTEs + // share a snapshot, so exclude our own still-visible attempt + // when deciding whether another writer remains. + let row = tx + .query_one(Statement::from_sql_and_values( + DbBackend::Postgres, + "WITH locked AS MATERIALIZED ( + SELECT space, value FROM kv_orphan_candidate + WHERE space = $1 AND value = $2 AND generation = $4 + AND deleting_at IS NULL + FOR UPDATE + ), refs AS MATERIALIZED ( + SELECT EXISTS( + SELECT 1 FROM kv_write WHERE space = $1 AND value = $2 AND generation = $4 + ) OR EXISTS( + SELECT 1 FROM current_kv WHERE space = $1 AND value = $2 AND generation = $4 + ) AS present + ), removed AS ( + DELETE FROM kv_orphan_attempt + WHERE space = $1 AND value = $2 AND attempt_id = $3 + AND generation = $4 + AND EXISTS(SELECT 1 FROM locked) + AND (SELECT present FROM refs) + RETURNING space, value + ), retired AS ( + DELETE FROM kv_orphan_candidate AS fence + USING removed + WHERE fence.space = removed.space + AND fence.value = removed.value + AND fence.generation = $4 + AND NOT EXISTS( + SELECT 1 FROM kv_orphan_attempt AS other + WHERE other.space = fence.space + AND other.value = fence.value + AND other.generation = $4 + AND other.attempt_id <> $3 + ) + RETURNING 1 + ) + SELECT (SELECT COUNT(*) FROM removed) AS removed, + (SELECT present FROM refs) AS referenced, + (SELECT COUNT(*) FROM retired) AS retired", + [key.into(), (*value).into(), attempt_id.to_owned().into(), (*generation).into()], + )) + .await? + .ok_or_else(|| DbErr::Custom("KV candidate finish returned no row".into()))?; + if !row.try_get::("", "referenced")? { + return Err(DbErr::Custom("KV blob has no committed reference".into())); + } + if row.try_get::("", "removed")? != 1 { + return Err(DbErr::Custom( + "KV blob attempt disappeared before commit".into(), + )); + } + continue; + } + let removed = attempt::Entity::delete_many() + .filter(attempt::Column::Space.eq(key.clone())) + .filter(attempt::Column::Value.eq(*value)) + .filter(attempt::Column::AttemptId.eq(attempt_id)) + .filter(attempt::Column::Generation.eq(*generation)) .exec(tx) .await?; + if removed.rows_affected != 1 { + return Err(DbErr::Custom( + "KV blob attempt disappeared before commit".into(), + )); + } + if !has_attempts(tx, &key, *value, *generation).await? + && has_references(tx, &key, *value, *generation).await? + { + candidate::Entity::delete_many() + .filter(candidate::Column::Space.eq(key)) + .filter(candidate::Column::Value.eq(*value)) + .filter(candidate::Column::Generation.eq(*generation)) + .exec(tx) + .await?; + } } Ok(()) } -/// A database operation or object-store deletion failed; the candidate is -/// retained for a future collection pass on rollback. #[derive(Debug, thiserror::Error)] pub enum CollectError { #[error("orphan candidate database operation failed: {0}")] Database(#[from] DbErr), + #[error("orphan blob metadata read failed: {0}")] + Read(String), #[error("orphan blob deletion failed: {0}")] Store(E), } -async fn remove_if_unreferenced( - tx: &DatabaseTransaction, - store: &S, - row: candidate::Model, -) -> Result> { +async fn has_attempts( + conn: &C, + space: &SpaceIdWrap, + value: Hash, + generation: i64, +) -> Result { + let query = attempt::Entity::find() + .filter(attempt::Column::Space.eq(space.clone())) + .filter(attempt::Column::Value.eq(value)) + .filter(attempt::Column::Generation.eq(generation)); + let query = if conn.get_database_backend() == DbBackend::MySql { + query.lock_exclusive() + } else { + query + }; + Ok(query.one(conn).await?.is_some()) +} + +async fn has_references( + conn: &C, + space: &SpaceIdWrap, + value: Hash, + generation: i64, +) -> Result { let historical = kv_write::Entity::find() - .filter(kv_write::Column::Space.eq(row.space.clone())) - .filter(kv_write::Column::Value.eq(row.value)) + .filter(kv_write::Column::Space.eq(space.clone())) + .filter(kv_write::Column::Value.eq(value)) + .filter(kv_write::Column::Generation.eq(generation)) .select_only() .column(kv_write::Column::Seq); let current = current_kv::Entity::find() - .filter(current_kv::Column::Space.eq(row.space.clone())) - .filter(current_kv::Column::Value.eq(row.value)) + .filter(current_kv::Column::Space.eq(space.clone())) + .filter(current_kv::Column::Value.eq(value)) + .filter(current_kv::Column::Generation.eq(generation)) .select_only() .column(current_kv::Column::Seq); - // MySQL's REPEATABLE READ snapshot may precede the writer whose hash - // fence we just acquired. Locking reads see its committed references. - let (historical, current) = if tx.get_database_backend() == DbBackend::MySql { + // MySQL's repeatable-read snapshot could precede the writer whose fence + // we just acquired. Locking reads see its committed references. + let (historical, current) = if conn.get_database_backend() == DbBackend::MySql { (historical.lock_exclusive(), current.lock_exclusive()) } else { (historical, current) }; - let referenced = historical.into_tuple::().one(tx).await?.is_some() - || current.into_tuple::().one(tx).await?.is_some(); - let removed = if referenced { - false - } else { - store - .remove(&row.space.0, &row.value) + Ok(historical.into_tuple::().one(conn).await?.is_some() + || current.into_tuple::().one(conn).await?.is_some()) +} + +async fn database_now(conn: &C) -> Result { + let sql = match conn.get_database_backend() { + DbBackend::Sqlite => "SELECT CAST(strftime('%s','now') AS INTEGER) AS now_epoch", + DbBackend::Postgres => { + "SELECT CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS BIGINT) AS now_epoch" + } + DbBackend::MySql => "SELECT CAST(UNIX_TIMESTAMP(CURRENT_TIMESTAMP) AS SIGNED) AS now_epoch", + }; + let row = conn + .query_one(Statement::from_string(conn.get_database_backend(), sql)) + .await? + .ok_or_else(|| DbErr::Custom("database clock unavailable".into()))?; + let epoch: i64 = row.try_get("", "now_epoch")?; + OffsetDateTime::from_unix_timestamp(epoch).map_err(|e| DbErr::Custom(e.to_string())) +} + +async fn lock_fence( + tx: &DatabaseTransaction, + space: &SpaceIdWrap, + value: Hash, + generation: i64, + skip_locked: bool, +) -> Result, DbErr> { + if tx.get_database_backend() == DbBackend::Sqlite { + let updated = candidate::Entity::update_many() + .col_expr( + candidate::Column::CreatedAt, + Expr::col(candidate::Column::CreatedAt).into(), + ) + .filter(candidate::Column::Space.eq(space.clone())) + .filter(candidate::Column::Value.eq(value)) + .filter(candidate::Column::Generation.eq(generation)) + .exec(tx) + .await?; + if updated.rows_affected == 0 { + return Ok(None); + } + candidate::Entity::find_by_id((space.clone(), value)) + .filter(candidate::Column::Generation.eq(generation)) + .one(tx) .await - .map_err(CollectError::Store)? - .is_some() + } else { + let query = candidate::Entity::find_by_id((space.clone(), value)) + .filter(candidate::Column::Generation.eq(generation)); + let query = if skip_locked && tx.get_database_backend() == DbBackend::Postgres { + query.lock_with_behavior(LockType::Update, LockBehavior::SkipLocked) + } else { + query.lock_exclusive() + }; + query.one(tx).await + } +} + +/// Drop one durable attempt and, if it is the last unreferenced claim, commit +/// the deleting fence and its byte entitlement before external I/O. +async fn prepare_deletion( + conn: &C, + store: &S, + space: &SpaceId, + value: Hash, + generation: i64, + attempt_id: &str, + skip_locked: bool, +) -> Result::Error>> +where + C: ConnectionTrait + TransactionTrait, + S: ImmutableDeleteStore + ImmutableReadStore, +{ + let bytes = store + .read(space, &value, generation) + .await + .map_err(|error| CollectError::Read(error.to_string()))? + .map_or(0, |content| content.len()); + let bytes = + i64::try_from(bytes).map_err(|_| DbErr::Custom("orphan blob size exceeds i64".into()))?; + let now = database_now(conn).await?; + let tx = conn.begin().await?; + let key = SpaceIdWrap(space.clone()); + let Some(fence) = lock_fence(&tx, &key, value, generation, skip_locked).await? else { + tx.commit().await?; + return Ok(false); }; - candidate::Entity::delete_by_id((row.space, row.value)) - .exec(tx) + if fence.deleting_at.is_some() { + tx.commit().await?; + return Ok(false); + } + let removed = attempt::Entity::delete_many() + .filter(attempt::Column::Space.eq(key.clone())) + .filter(attempt::Column::Value.eq(value)) + .filter(attempt::Column::AttemptId.eq(attempt_id)) + .filter(attempt::Column::Generation.eq(generation)) + .exec(&tx) .await?; - Ok(removed) + if removed.rows_affected == 0 { + tx.commit().await?; + return Ok(false); + } + if has_attempts(&tx, &key, value, generation).await? { + tx.commit().await?; + return Ok(false); + } + if has_references(&tx, &key, value, generation).await? { + candidate::Entity::delete_many() + .filter(candidate::Column::Space.eq(key)) + .filter(candidate::Column::Value.eq(value)) + .filter(candidate::Column::Generation.eq(generation)) + .exec(&tx) + .await?; + tx.commit().await?; + return Ok(false); + } + candidate::Entity::update_many() + .col_expr(candidate::Column::DeletingAt, Expr::value(now)) + .col_expr(candidate::Column::DeletionBytes, Expr::value(bytes)) + .filter(candidate::Column::Space.eq(key)) + .filter(candidate::Column::Value.eq(value)) + .filter(candidate::Column::Generation.eq(generation)) + .exec(&tx) + .await?; + tx.commit().await?; + Ok(true) } -/// Refund a known failed invocation before returning to its caller. The -/// durable candidate remains for periodic recovery if deletion fails. -pub async fn refund_failed( +/// Complete a committed deleting fence, with no DB transaction or checked-out +/// DB connection during the object-store call. A lost response leaves it for GC. +async fn complete_deletion( conn: &C, store: &S, - blobs: &[(SpaceId, Hash)], -) -> Result<(), CollectError> + space: &SpaceId, + value: Hash, + generation: i64, +) -> Result::Error>> where C: ConnectionTrait + TransactionTrait, S: ImmutableDeleteStore, { - for (space, value) in ordered_unique(blobs).iter() { - let tx = conn.begin().await?; - let key = (SpaceIdWrap(space.clone()), *value); - let row = if tx.get_database_backend() == DbBackend::Sqlite { - let updated = candidate::Entity::update_many() + let removed = store + .remove(space, &value, generation) + .await + .map_err(CollectError::Store)? + .is_some(); + // Registration locks ledger before fence; use that same order to retire + // exactly the deleting generation and allocate the next storage key. + let tx = conn.begin().await?; + let key = SpaceIdWrap(space.clone()); + let current = if tx.get_database_backend() == DbBackend::Sqlite { + ledger::Entity::update_many() + .col_expr( + ledger::Column::Generation, + Expr::col(ledger::Column::Generation).into(), + ) + .filter(ledger::Column::Space.eq(key.clone())) + .filter(ledger::Column::Value.eq(value)) + .exec(&tx) + .await?; + ledger::Entity::find_by_id((key.clone(), value)) + .one(&tx) + .await? + } else { + ledger::Entity::find_by_id((key.clone(), value)) + .lock_exclusive() + .one(&tx) + .await? + }; + if current.is_some_and(|row| row.generation == generation) { + let retired = candidate::Entity::delete_many() + .filter(candidate::Column::Space.eq(key.clone())) + .filter(candidate::Column::Value.eq(value)) + .filter(candidate::Column::Generation.eq(generation)) + .filter(candidate::Column::DeletingAt.is_not_null()) + .exec(&tx) + .await?; + if retired.rows_affected == 1 { + ledger::Entity::update_many() .col_expr( - candidate::Column::CreatedAt, - Expr::col(candidate::Column::CreatedAt).into(), + ledger::Column::Generation, + Expr::value( + generation + .checked_add(1) + .ok_or_else(|| DbErr::Custom("KV blob generation exhausted".into()))?, + ), ) - .filter(candidate::Column::Space.eq(key.0.clone())) - .filter(candidate::Column::Value.eq(key.1)) + .filter(ledger::Column::Space.eq(key)) + .filter(ledger::Column::Value.eq(value)) + .filter(ledger::Column::Generation.eq(generation)) .exec(&tx) .await?; - if updated.rows_affected == 0 { - continue; - } - candidate::Entity::find_by_id(key).one(&tx).await? - } else { - candidate::Entity::find_by_id(key) - .lock_exclusive() - .one(&tx) - .await? - }; - if let Some(row) = row { - remove_if_unreferenced(&tx, store, row).await?; } - tx.commit().await?; + } + tx.commit().await?; + Ok(removed) +} + +/// Pending refunds used while reconstructing object-store quota after restart. +/// The storage listing must intersect these keys with objects it actually saw. +pub async fn deleting_fences( + conn: &C, +) -> Result, DbErr> { + Ok(candidate::Entity::find() + .filter(candidate::Column::DeletingAt.is_not_null()) + .order_by_asc(candidate::Column::Space) + .order_by_asc(candidate::Column::Value) + .all(conn) + .await? + .into_iter() + .map(|row| (row.space.0, row.value, row.generation)) + .collect()) +} + +async fn defer_deletion( + conn: &C, + space: &SpaceId, + value: Hash, + generation: i64, +) -> Result<(), DbErr> { + let retry_at = database_now(conn).await? + Duration::seconds(59); + candidate::Entity::update_many() + .col_expr(candidate::Column::DeletingAt, Expr::value(retry_at)) + .filter(candidate::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(candidate::Column::Value.eq(value)) + .filter(candidate::Column::Generation.eq(generation)) + .filter(candidate::Column::DeletingAt.is_not_null()) + .exec(conn) + .await?; + Ok(()) +} + +async fn defer_attempt( + conn: &C, + space: &SpaceId, + value: Hash, + generation: i64, + attempt_id: &str, +) -> Result<(), DbErr> { + attempt::Entity::update_many() + .col_expr( + attempt::Column::CreatedAt, + Expr::value(database_now(conn).await?), + ) + .filter(attempt::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(attempt::Column::Value.eq(value)) + .filter(attempt::Column::Generation.eq(generation)) + .filter(attempt::Column::AttemptId.eq(attempt_id)) + .exec(conn) + .await?; + Ok(()) +} +pub async fn refund_failed( + conn: &C, + store: &S, + blobs: &[(SpaceId, Hash, i64)], + attempt_id: &str, +) -> Result<(), CollectError<::Error>> +where + C: ConnectionTrait + TransactionTrait, + S: ImmutableDeleteStore + ImmutableReadStore, +{ + for (space, value, generation) in blobs { + if prepare_deletion(conn, store, space, *value, *generation, attempt_id, false).await? { + complete_deletion(conn, store, space, *value, *generation).await?; + } } Ok(()) } @@ -207,88 +743,105 @@ pub async fn collect_candidates( store: &S, max: u64, min_age: Duration, -) -> Result> +) -> Result::Error>> where C: ConnectionTrait + TransactionTrait, - S: ImmutableDeleteStore, + S: ImmutableDeleteStore + ImmutableReadStore, { Ok(collect_candidates_with_progress(conn, store, max, min_age) .await? .0) } -/// Collect at most `min(max, 256)` old candidates in one pass. -/// -/// A row lock excludes uploaders of the same `(space, hash)` until reference -/// checks, blob deletion and candidate removal have completed. Historical -/// `kv_write` references are as important as current KV references: a sync -/// client may still read an older value. +/// Reclaim expired attempts and retry committed deleting fences. Every +/// candidate is handled in a separate short transaction; slow S3 never holds +/// a fence or database connection. pub async fn collect_candidates_with_progress( conn: &C, store: &S, max: u64, min_age: Duration, -) -> Result<(usize, usize), CollectError> +) -> Result<(usize, usize), CollectError<::Error>> where C: ConnectionTrait + TransactionTrait, - S: ImmutableDeleteStore, + S: ImmutableDeleteStore + ImmutableReadStore, { let limit = max.min(MAX_BATCH_SIZE); if limit == 0 { return Ok((0, 0)); } - let cutoff = OffsetDateTime::now_utc() + let now = database_now(conn).await?; + let cutoff = now .checked_sub(min_age) .unwrap_or(OffsetDateTime::UNIX_EPOCH); - let tx = conn.begin().await?; - let query = candidate::Entity::find() - .filter(candidate::Column::CreatedAt.lte(cutoff)) - .order_by_asc(candidate::Column::CreatedAt) + let deleting = candidate::Entity::find() + .filter(candidate::Column::DeletingAt.is_not_null()) + .filter(candidate::Column::DeletingAt.lte(now)) + .order_by_asc(candidate::Column::DeletingAt) .order_by_asc(candidate::Column::Space) .order_by_asc(candidate::Column::Value) - .limit(limit); - let query = if tx.get_database_backend() == DbBackend::Postgres { - query.lock_with_behavior(LockType::Update, LockBehavior::SkipLocked) - } else { - query - }; - let rows = query.all(&tx).await?; - let scanned = rows.len(); + .limit(limit) + .all(conn) + .await?; + let attempts = attempt::Entity::find() + .filter(attempt::Column::CreatedAt.lte(cutoff)) + .order_by_asc(attempt::Column::CreatedAt) + .order_by_asc(attempt::Column::Space) + .order_by_asc(attempt::Column::Value) + .limit(limit.saturating_sub(deleting.len() as u64)) + .all(conn) + .await?; + let scanned = deleting.len() + attempts.len(); let mut removed = 0; - for row in rows { - // PostgreSQL SELECT FOR UPDATE already owns this lock. MySQL - // obtains a fresh locking read; SQLite upgrades to a write lock. - if tx.get_database_backend() != DbBackend::Postgres { - let fresh = if tx.get_database_backend() == DbBackend::Sqlite { - let updated = candidate::Entity::update_many() - .col_expr( - candidate::Column::CreatedAt, - Expr::col(candidate::Column::CreatedAt).into(), - ) - .filter(candidate::Column::Space.eq(row.space.clone())) - .filter(candidate::Column::Value.eq(row.value)) - .exec(&tx) - .await?; - if updated.rows_affected == 0 { - continue; + let mut first_error = None; + for row in deleting { + match complete_deletion(conn, store, &row.space.0, row.value, row.generation).await { + Ok(did_remove) => removed += usize::from(did_remove), + Err(error) => { + defer_deletion(conn, &row.space.0, row.value, row.generation).await?; + first_error.get_or_insert(error); + } + } + } + for row in attempts { + match prepare_deletion( + conn, + store, + &row.space.0, + row.value, + row.generation, + &row.attempt_id, + true, + ) + .await + { + Ok(true) => { + match complete_deletion(conn, store, &row.space.0, row.value, row.generation).await + { + Ok(did_remove) => removed += usize::from(did_remove), + Err(error) => { + defer_deletion(conn, &row.space.0, row.value, row.generation).await?; + first_error.get_or_insert(error); + } } - candidate::Entity::find_by_id((row.space.clone(), row.value)) - .one(&tx) - .await? - } else { - candidate::Entity::find_by_id((row.space.clone(), row.value)) - .lock_exclusive() - .one(&tx) - .await? - }; - if !fresh.is_some_and(|candidate| candidate.created_at <= cutoff) { - continue; + } + Ok(false) => {} + Err(error) => { + defer_attempt( + conn, + &row.space.0, + row.value, + row.generation, + &row.attempt_id, + ) + .await?; + first_error.get_or_insert(error); } } - - removed += usize::from(remove_if_unreferenced(&tx, store, row).await?); } - tx.commit().await?; + if let Some(error) = first_error { + return Err(error); + } Ok((removed, scanned)) } @@ -296,7 +849,7 @@ where mod tests { use super::*; use crate::hash::hash; - use crate::models::kv_orphan_candidate; + use crate::models::{kv_orphan_attempt, kv_orphan_candidate}; use crate::storage::{ memory::MemoryStore, HashBuffer, ImmutableReadStore, ImmutableWriteStore, StoreSize, }; @@ -306,7 +859,100 @@ mod tests { EntityTrait, QueryFilter, Statement, TransactionTrait, }; use sea_orm_migration::MigratorTrait; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; use tinycloud_auth::{resolver::DID_METHODS, ssi::jwk::JWK}; + use tokio::sync::Notify; + + #[derive(Clone)] + struct InterruptedDelete { + inner: Arc, + entered: Arc, + resume: Arc, + pause: Arc, + lose_response: Arc, + fail_read: Arc, + } + + impl InterruptedDelete { + fn new(inner: Arc) -> Self { + Self { + inner, + entered: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + pause: Arc::new(AtomicBool::new(false)), + lose_response: Arc::new(AtomicBool::new(false)), + fail_read: Arc::new(AtomicBool::new(false)), + } + } + } + + #[async_trait::async_trait] + impl ImmutableReadStore for InterruptedDelete { + type Error = std::io::Error; + type Readable = ::Readable; + + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { + self.inner.contains(space, id, generation).await + } + + async fn read( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result>, Self::Error> { + if self.fail_read.swap(false, Ordering::SeqCst) { + return Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "lost HEAD response", + )); + } + self.inner.read(space, id, generation).await + } + + async fn read_range( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + range: crate::storage::ByteRangeSpec, + ) -> Result>, Self::Error> { + self.inner.read_range(space, id, generation, range).await + } + } + + #[async_trait::async_trait] + impl ImmutableDeleteStore for InterruptedDelete { + type Error = std::io::Error; + + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { + if self.pause.swap(false, Ordering::SeqCst) { + self.entered.notify_one(); + self.resume.notified().await; + } + let removed = self.inner.remove(space, id, generation).await?; + if self.lose_response.swap(false, Ordering::SeqCst) { + return Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "lost DELETE response", + )); + } + Ok(removed) + } + } async fn database() -> DatabaseConnection { let db = Database::connect(ConnectOptions::new("sqlite::memory:".to_owned())) @@ -329,39 +975,81 @@ mod tests { } async fn upload(store: &MemoryStore, space: &SpaceId, bytes: &[u8]) -> Hash { + upload_generation(store, space, bytes, 0).await + } + + async fn upload_generation( + store: &MemoryStore, + space: &SpaceId, + bytes: &[u8], + generation: i64, + ) -> Hash { let mut staged = HashBuffer::new(Vec::new()); staged.write_all(bytes).await.unwrap(); - store.persist(space, staged).await.unwrap() + store.persist(space, staged, generation).await.unwrap() } async fn age(db: &DatabaseConnection, space: &SpaceId, value: Hash) { - kv_orphan_candidate::Entity::update_many() + kv_orphan_attempt::Entity::update_many() .col_expr( - kv_orphan_candidate::Column::CreatedAt, + kv_orphan_attempt::Column::CreatedAt, Expr::value(time::OffsetDateTime::now_utc() - time::Duration::hours(2)), ) - .filter(kv_orphan_candidate::Column::Space.eq(crate::types::SpaceIdWrap(space.clone()))) - .filter(kv_orphan_candidate::Column::Value.eq(value)) + .filter(kv_orphan_attempt::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(kv_orphan_attempt::Column::Value.eq(value)) .exec(db) .await .unwrap(); } - async fn reference(db: &DatabaseConnection, table: &str, space: &SpaceId, value: Hash) { - let sql = if table == "kv_write" { - "INSERT INTO kv_write (space, key, invocation, seq, epoch, epoch_seq, value, metadata) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}')" - } else { - "INSERT INTO current_kv (space, key, invocation, seq, epoch, epoch_seq, value, metadata, deleted) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}', 0)" + async fn register( + db: &DatabaseConnection, + blobs: &[(SpaceId, Hash)], + id: &str, + ) -> Vec<(SpaceId, Hash, i64)> { + let tx = db.begin().await.unwrap(); + let registered = register_candidates(&tx, blobs, id).await.unwrap(); + tx.commit().await.unwrap(); + registered + } + + fn gen0(blobs: &[(SpaceId, Hash)]) -> Vec<(SpaceId, Hash, i64)> { + blobs + .iter() + .map(|(space, value)| (space.clone(), *value, 0)) + .collect() + } + + async fn reference( + db: &C, + table: &str, + space: &SpaceId, + value: Hash, + generation: i64, + ) { + let sql = match (table, generation) { + ("kv_write", 0) => + "INSERT INTO kv_write (space, key, invocation, seq, epoch, epoch_seq, value, metadata) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}')", + ("kv_write", _) => + "INSERT INTO kv_write (space, key, invocation, seq, epoch, epoch_seq, value, metadata, generation) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}', ?)", + (_, 0) => + "INSERT INTO current_kv (space, key, invocation, seq, epoch, epoch_seq, value, metadata, deleted) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}', 0)", + (_, _) => + "INSERT INTO current_kv (space, key, invocation, seq, epoch, epoch_seq, value, metadata, deleted, generation) VALUES (?, 'a', ?, 1, ?, 1, ?, '{}', 0, ?)", }; + let mut values: Vec = vec![ + space.to_string().into(), + Vec::::from(hash(table.as_bytes())).into(), + Vec::::from(hash(b"epoch")).into(), + Vec::::from(value).into(), + ]; + if generation != 0 { + values.push(generation.into()); + } db.execute(Statement::from_sql_and_values( DbBackend::Sqlite, sql, - [ - space.to_string().into(), - Vec::::from(hash(table.as_bytes())).into(), - Vec::::from(hash(b"epoch")).into(), - Vec::::from(value).into(), - ], + values, )) .await .unwrap(); @@ -379,12 +1067,30 @@ mod tests { (history_space.clone(), historical), (current_space.clone(), current), ]; - register_candidates(&db, &candidates).await.unwrap(); + register(&db, &candidates, "a").await; for (space, hash) in &candidates { age(&db, space, *hash).await; } - reference(&db, "kv_write", &history_space, historical).await; - reference(&db, "current_kv", ¤t_space, current).await; + reference(&db, "kv_write", &history_space, historical, 0).await; + reference(&db, "current_kv", ¤t_space, current, 0).await; + assert_eq!( + kv_write::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap() + .generation, + 0 + ); + assert_eq!( + current_kv::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap() + .generation, + 0 + ); assert_eq!( collect_candidates(&db, &store, 10, time::Duration::hours(1)) @@ -393,7 +1099,7 @@ mod tests { 0 ); for (space, hash) in &candidates { - assert!(store.contains(space, hash).await.unwrap()); + assert!(store.contains(space, hash, 0).await.unwrap()); assert!(kv_orphan_candidate::Entity::find_by_id(( crate::types::SpaceIdWrap(space.clone()), *hash @@ -411,16 +1117,14 @@ mod tests { let store = MemoryStore::default(); let space = space(); let value = hash(b"failed-precondition"); - register_candidates(&db, &[(space.clone(), value)]) - .await - .unwrap(); + register(&db, &[(space.clone(), value)], "a").await; assert_eq!(upload(&store, &space, b"failed-precondition").await, value); assert_eq!( store.total_size(&space).await.unwrap(), Some(b"failed-precondition".len() as u64) ); let tx = db.begin().await.unwrap(); - lock_candidates(&tx, &[(space.clone(), value)]) + lock_candidates(&tx, &[(space.clone(), value, 0)], "a") .await .unwrap(); tx.rollback().await.unwrap(); @@ -432,7 +1136,7 @@ mod tests { .unwrap(), 1 ); - assert!(!store.contains(&space, &value).await.unwrap()); + assert!(!store.contains(&space, &value, 0).await.unwrap()); assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); assert!( kv_orphan_candidate::Entity::find_by_id((crate::types::SpaceIdWrap(space), value)) @@ -451,16 +1155,12 @@ mod tests { let mut old = Vec::new(); for bytes in [b"old-1".as_slice(), b"old-2", b"old-3"] { let value = upload(&store, &space, bytes).await; - register_candidates(&db, &[(space.clone(), value)]) - .await - .unwrap(); + register(&db, &[(space.clone(), value)], "a").await; age(&db, &space, value).await; old.push(value); } let fresh = upload(&store, &space, b"fresh").await; - register_candidates(&db, &[(space.clone(), fresh)]) - .await - .unwrap(); + register(&db, &[(space.clone(), fresh)], "a").await; assert_eq!( collect_candidates(&db, &store, 2, time::Duration::hours(1)) .await @@ -469,10 +1169,10 @@ mod tests { ); let mut surviving = 0; for value in &old { - surviving += usize::from(store.contains(&space, value).await.unwrap()); + surviving += usize::from(store.contains(&space, value, 0).await.unwrap()); } assert_eq!(surviving, 1); - assert!(store.contains(&space, &fresh).await.unwrap()); + assert!(store.contains(&space, &fresh, 0).await.unwrap()); assert_eq!( collect_candidates(&db, &store, 2, time::Duration::hours(1)) .await @@ -485,7 +1185,7 @@ mod tests { .unwrap(), 0 ); - assert!(store.contains(&space, &fresh).await.unwrap()); + assert!(store.contains(&space, &fresh, 0).await.unwrap()); } #[tokio::test] @@ -494,35 +1194,216 @@ mod tests { let store = MemoryStore::default(); let space = space(); let value = upload(&store, &space, b"same-hash").await; - register_candidates(&db, &[(space.clone(), value)]) + register(&db, &[(space.clone(), value)], "a").await; + age(&db, &space, value).await; + register(&db, &[(space.clone(), value)], "b").await; + + assert_eq!( + collect_candidates(&db, &store, 1, time::Duration::hours(1)) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &value, 0).await.unwrap()); + } + + #[tokio::test] + async fn first_failure_preserves_second_attempt_and_its_rollback_cleanup() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = hash(b"same candidate race"); + let blobs = [(space.clone(), value)]; + let tx = db.begin().await.unwrap(); + register_candidates(&tx, &blobs, "a").await.unwrap(); + tx.commit().await.unwrap(); + assert_eq!(upload(&store, &space, b"same candidate race").await, value); + let tx = db.begin().await.unwrap(); + register_candidates(&tx, &blobs, "b").await.unwrap(); + tx.commit().await.unwrap(); + + refund_failed(&db, &store, &gen0(&blobs), "a") .await .unwrap(); - age(&db, &space, value).await; - register_candidates(&db, &[(space.clone(), value)]) + assert!(store.contains(&space, &value, 0).await.unwrap()); + let b = db.begin().await.unwrap(); + lock_candidates(&b, &gen0(&blobs), "b").await.unwrap(); + b.rollback().await.unwrap(); + refund_failed(&db, &store, &gen0(&blobs), "b") .await .unwrap(); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + #[tokio::test] + async fn successful_same_hash_writer_protects_failed_writers_blob() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"referenced-dedup").await; + let blobs = [(space.clone(), value)]; + register(&db, &blobs, "a").await; + register(&db, &blobs, "b").await; + let tx = db.begin().await.unwrap(); + lock_candidates(&tx, &gen0(&blobs), "b").await.unwrap(); + reference(&tx, "kv_write", &space, value, 0).await; + finish_candidates(&tx, &gen0(&blobs), "b").await.unwrap(); + tx.commit().await.unwrap(); + refund_failed(&db, &store, &gen0(&blobs), "a") + .await + .unwrap(); + assert!(store.contains(&space, &value, 0).await.unwrap()); assert_eq!( - collect_candidates(&db, &store, 1, time::Duration::hours(1)) + store.total_size(&space).await.unwrap(), + Some(b"referenced-dedup".len() as u64) + ); + assert_eq!( + collect_candidates(&db, &store, 10, Duration::ZERO) .await .unwrap(), 0 ); - assert!(store.contains(&space, &value).await.unwrap()); + assert!( + kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space), value)) + .one(&db) + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn replayed_registration_requires_own_attempt_without_duplicating_it() { + let db = database().await; + let space = space(); + let value = hash(b"replay"); + let blobs = [(space.clone(), value)]; + register(&db, &blobs, "a").await; + register(&db, &blobs, "a").await; + let attempts = kv_orphan_attempt::Entity::find() + .filter(kv_orphan_attempt::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(kv_orphan_attempt::Column::Value.eq(value)) + .all(&db) + .await + .unwrap(); + assert_eq!(attempts.len(), 1); + let tx = db.begin().await.unwrap(); + let error = lock_candidates(&tx, &gen0(&blobs), "b").await.unwrap_err(); + assert!(is_candidate_unavailable(&error)); + tx.rollback().await.unwrap(); + } + + #[tokio::test] + async fn deleting_fence_refuses_new_attempt_until_delete_finishes() { + let db = database().await; + let space = space(); + let value = hash(b"stale delete cannot hit a new write"); + let blobs = [(space.clone(), value)]; + let tx = db.begin().await.unwrap(); + register_candidates(&tx, &blobs, "a").await.unwrap(); + tx.commit().await.unwrap(); + kv_orphan_candidate::Entity::update_many() + .col_expr( + kv_orphan_candidate::Column::DeletingAt, + Expr::value(time::OffsetDateTime::now_utc()), + ) + .filter(kv_orphan_candidate::Column::Space.eq(SpaceIdWrap(space))) + .filter(kv_orphan_candidate::Column::Value.eq(value)) + .exec(&db) + .await + .unwrap(); + let tx = db.begin().await.unwrap(); + assert!(register_candidates(&tx, &blobs, "b").await.is_err()); + tx.rollback().await.unwrap(); + } + + #[tokio::test] + async fn slow_delete_releases_database_and_refuses_new_preflight() { + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let value = upload(&inner, &space, b"slow-delete").await; + let blobs = [(space.clone(), value)]; + let old = register(&db, &blobs, "a").await; + store.pause.store(true, Ordering::SeqCst); + let deleting = tokio::spawn({ + let db = db.clone(); + let store = store.clone(); + let old = old.clone(); + async move { refund_failed(&db, &store, &old, "a").await } + }); + store.entered.notified().await; + let fence = kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space.clone()), value)) + .one(&db) + .await + .unwrap() + .unwrap(); + assert_eq!(fence.deletion_bytes, Some(b"slow-delete".len() as i64)); + assert!(fence.deleting_at.is_some()); + let tx = tokio::time::timeout(std::time::Duration::from_secs(1), db.begin()) + .await + .unwrap() + .unwrap(); + let error = register_candidates(&tx, &blobs, "b").await.unwrap_err(); + assert!(is_candidate_unavailable(&error)); + tx.rollback().await.unwrap(); + store.resume.notify_one(); + deleting.await.unwrap().unwrap(); + assert!(!inner.contains(&space, &value, 0).await.unwrap()); + let next = register(&db, &blobs, "b").await; + assert_eq!(next[0].2, 1); + assert_eq!( + upload_generation(&inner, &space, b"slow-delete", 1).await, + value + ); + let tx = db.begin().await.unwrap(); + lock_candidates(&tx, &next, "b").await.unwrap(); + reference(&tx, "kv_write", &space, value, 1).await; + finish_candidates(&tx, &next, "b").await.unwrap(); + tx.commit().await.unwrap(); + assert!(kv_write::Entity::find() + .filter(kv_write::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(kv_write::Column::Value.eq(value)) + .one(&db) + .await + .unwrap() + .is_some()); + assert!(inner.contains(&space, &value, 1).await.unwrap()); + assert_eq!( + inner.total_size(&space).await.unwrap(), + Some(b"slow-delete".len() as u64) + ); } #[tokio::test] - async fn uploader_recreates_candidate_deleted_by_earlier_same_hash_commit() { + async fn lost_delete_response_retries_from_durable_fence_once() { let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); let space = space(); - let value = hash(b"same-hash"); + let value = upload(&inner, &space, b"lost-delete").await; let blobs = [(space.clone(), value)]; - register_candidates(&db, &blobs).await.unwrap(); - // The next uploader has already pre-registered this same hash. - let earlier = db.begin().await.unwrap(); - lock_candidates(&earlier, &blobs).await.unwrap(); - finish_candidates(&earlier, &blobs).await.unwrap(); - earlier.commit().await.unwrap(); + let old = register(&db, &blobs, "a").await; + store.lose_response.store(true, Ordering::SeqCst); + assert!(refund_failed(&db, &store, &old, "a").await.is_err()); + let fence = kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space.clone()), value)) + .one(&db) + .await + .unwrap() + .unwrap(); + assert!(fence.deleting_at.is_some()); + assert_eq!(fence.deletion_bytes, Some(b"lost-delete".len() as i64)); + assert!(!inner.contains(&space, &value, 0).await.unwrap()); + assert_eq!(deleting_fences(&db).await.unwrap(), old); + assert_eq!( + collect_candidates(&db, &store, 10, Duration::ZERO) + .await + .unwrap(), + 0, + "the retry confirms absence without charging a second refund" + ); assert!( kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space.clone()), value)) .one(&db) @@ -530,17 +1411,93 @@ mod tests { .unwrap() .is_none() ); - let later = db.begin().await.unwrap(); - lock_candidates(&later, &blobs).await.unwrap(); + assert!(deleting_fences(&db).await.unwrap().is_empty()); + assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn two_deleters_cannot_remove_a_newly_committed_same_hash() { + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let value = upload(&inner, &space, b"two-deleter-race").await; + let blobs = [(space.clone(), value)]; + let old = register(&db, &blobs, "a").await; + + store.pause.store(true, Ordering::SeqCst); + let first = tokio::spawn({ + let db = db.clone(); + let store = store.clone(); + let old = old.clone(); + async move { refund_failed(&db, &store, &old, "a").await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), store.entered.notified()) + .await + .unwrap(); + store.pause.store(true, Ordering::SeqCst); + let second_entered = store.entered.notified(); + let second = tokio::spawn({ + let db = db.clone(); + let store = store.clone(); + async move { collect_candidates(&db, &store, 10, Duration::ZERO).await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), second_entered) + .await + .unwrap(); + store.resume.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(5), first) + .await + .unwrap() + .unwrap() + .unwrap(); + + let next = register(&db, &blobs, "b").await; + assert_eq!(next[0].2, 1); + let stale = db.begin().await.unwrap(); + assert!(is_candidate_unavailable( + &lock_candidates(&stale, &old, "b").await.unwrap_err() + )); + stale.rollback().await.unwrap(); + assert_eq!( + upload_generation(&inner, &space, b"two-deleter-race", 1).await, + value + ); + let tx = db.begin().await.unwrap(); + lock_candidates(&tx, &next, "b").await.unwrap(); + reference(&tx, "kv_write", &space, value, 1).await; + finish_candidates(&tx, &next, "b").await.unwrap(); + tx.commit().await.unwrap(); + + store.resume.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(5), second) + .await + .unwrap() + .unwrap() + .unwrap(); assert!( - kv_orphan_candidate::Entity::find_by_id((SpaceIdWrap(space), value)) - .one(&later) + inner.contains(&space, &value, 1).await.unwrap(), + "a delayed gen0 DELETE removed newly committed gen1 content" + ); + assert!(!inner.contains(&space, &value, 0).await.unwrap()); + assert_eq!( + kv_write::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap() + .generation, + 1 + ); + assert_eq!( + ledger::Entity::find_by_id((SpaceIdWrap(space), value)) + .one(&db) .await .unwrap() - .is_some(), - "the second upload must have a durable hash fence after the first commit" + .unwrap() + .generation, + 1 ); - later.commit().await.unwrap(); } #[tokio::test] @@ -555,21 +1512,23 @@ mod tests { let store = MemoryStore::default(); let space = space(); let value = upload(&store, &space, b"upload-in-progress").await; - register_candidates(&fixture.conn, &[(space.clone(), value)]) + let prep = fixture.conn.begin().await.unwrap(); + register_candidates(&prep, &[(space.clone(), value)], "a") .await .unwrap(); - kv_orphan_candidate::Entity::update_many() + prep.commit().await.unwrap(); + kv_orphan_attempt::Entity::update_many() .col_expr( - kv_orphan_candidate::Column::CreatedAt, + kv_orphan_attempt::Column::CreatedAt, Expr::value(time::OffsetDateTime::now_utc() - time::Duration::hours(2)), ) - .filter(kv_orphan_candidate::Column::Space.eq(crate::types::SpaceIdWrap(space.clone()))) - .filter(kv_orphan_candidate::Column::Value.eq(value)) + .filter(kv_orphan_attempt::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(kv_orphan_attempt::Column::Value.eq(value)) .exec(&fixture.conn) .await .unwrap(); let tx = fixture.conn.begin().await.unwrap(); - lock_candidates(&tx, &[(space.clone(), value)]) + lock_candidates(&tx, &[(space.clone(), value, 0)], "a") .await .unwrap(); let collected = tokio::time::timeout( @@ -580,7 +1539,7 @@ mod tests { .expect("collector should skip the locked candidate") .unwrap(); assert_eq!(collected, 0); - assert!(store.contains(&space, &value).await.unwrap()); + assert!(store.contains(&space, &value, 0).await.unwrap()); tx.rollback().await.unwrap(); assert_eq!( collect_candidates(&fixture.conn, &store, 1, time::Duration::hours(1)) @@ -590,21 +1549,300 @@ mod tests { ); } + #[tokio::test] + async fn postgres_failed_attempt_cannot_erase_another_durable_registration() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_failed_attempt_cannot_erase_another_durable_registration", + ) + .await + else { + return; + }; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"same-hash overlap").await; + let blobs = [(space.clone(), value)]; + register(&fixture.conn, &blobs, "a").await; + register(&fixture.conn, &blobs, "b").await; + refund_failed(&fixture.conn, &store, &gen0(&blobs), "a") + .await + .unwrap(); + assert!(store.contains(&space, &value, 0).await.unwrap()); + let tx = fixture.conn.begin().await.unwrap(); + lock_candidates(&tx, &gen0(&blobs), "b").await.unwrap(); + tx.rollback().await.unwrap(); + refund_failed(&fixture.conn, &store, &gen0(&blobs), "b") + .await + .unwrap(); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert!(deleting_fences(&fixture.conn).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn postgres_delete_in_flight_blocks_dedup_and_retries_lost_response() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_delete_in_flight_blocks_dedup_and_retries_lost_response", + ) + .await + else { + return; + }; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let value = upload(&inner, &space, b"delete-in-flight").await; + let blobs = [(space.clone(), value)]; + let old = register(&fixture.conn, &blobs, "a").await; + store.pause.store(true, Ordering::SeqCst); + let deleting = tokio::spawn({ + let conn = fixture.conn.clone(); + let store = store.clone(); + let old = old.clone(); + async move { refund_failed(&conn, &store, &old, "a").await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), store.entered.notified()) + .await + .unwrap(); + assert_eq!(deleting_fences(&fixture.conn).await.unwrap(), old); + let tx = fixture.conn.begin().await.unwrap(); + let unavailable = register_candidates(&tx, &blobs, "b").await.unwrap_err(); + assert!(is_candidate_unavailable(&unavailable)); + tx.rollback().await.unwrap(); + store.resume.notify_one(); + deleting.await.unwrap().unwrap(); + assert!(!inner.contains(&space, &value, 0).await.unwrap()); + let second = register(&fixture.conn, &blobs, "b").await; + assert_eq!(second[0].2, 1); + assert_eq!( + upload_generation(&inner, &space, b"delete-in-flight", 1).await, + value + ); + let tx = fixture.conn.begin().await.unwrap(); + lock_candidates(&tx, &second, "b").await.unwrap(); + tx.rollback().await.unwrap(); + assert!(inner.contains(&space, &value, 1).await.unwrap()); + refund_failed(&fixture.conn, &store, &second, "b") + .await + .unwrap(); + assert!(!inner.contains(&space, &value, 1).await.unwrap()); + + // A successful physical DELETE with a lost reply keeps only the + // deleting fence, which GC consumes without a second refund. + let third = register(&fixture.conn, &blobs, "c").await; + assert_eq!(third[0].2, 2); + assert_eq!( + upload_generation(&inner, &space, b"delete-in-flight", 2).await, + value + ); + store.lose_response.store(true, Ordering::SeqCst); + assert!(refund_failed(&fixture.conn, &store, &third, "c") + .await + .is_err()); + assert_eq!(deleting_fences(&fixture.conn).await.unwrap(), third); + assert!(!inner.contains(&space, &value, 2).await.unwrap()); + assert_eq!( + collect_candidates(&fixture.conn, &store, 10, Duration::ZERO) + .await + .unwrap(), + 0 + ); + assert!(deleting_fences(&fixture.conn).await.unwrap().is_empty()); + assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn postgres_generation_survives_independent_pool_and_delayed_delete() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_generation_survives_independent_pool_and_delayed_delete", + ) + .await + else { + return; + }; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let value = upload(&inner, &space, b"independent-pool").await; + let blobs = [(space.clone(), value)]; + let first = register(&fixture.conn, &blobs, "first").await; + assert_eq!(first[0].2, 0); + store.pause.store(true, Ordering::SeqCst); + let deletion = tokio::spawn({ + let conn = fixture.conn.clone(); + let store = store.clone(); + let first = first.clone(); + async move { refund_failed(&conn, &store, &first, "first").await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), store.entered.notified()) + .await + .unwrap(); + store.pause.store(true, Ordering::SeqCst); + let second_entered = store.entered.notified(); + let delayed = tokio::spawn({ + let conn = fixture.conn.clone(); + let store = store.clone(); + async move { collect_candidates(&conn, &store, 1, Duration::ZERO).await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), second_entered) + .await + .unwrap(); + store.resume.notify_one(); + deletion.await.unwrap().unwrap(); + let schema: String = fixture + .conn + .query_one(Statement::from_string( + DbBackend::Postgres, + "SELECT current_schema() AS schema_name", + )) + .await + .unwrap() + .unwrap() + .try_get("", "schema_name") + .unwrap(); + let url = crate::test_support::postgres_test_url( + "postgres_generation_survives_independent_pool_and_delayed_delete", + ) + .unwrap(); + let mut options = ConnectOptions::new(url); + options.set_schema_search_path(schema); + let restarted = Database::connect(options).await.unwrap(); + let second = register(&restarted, &blobs, "second").await; + assert_eq!(second[0].2, 1); + assert_eq!( + upload_generation(&inner, &space, b"independent-pool", 1).await, + value + ); + let tx = restarted.begin().await.unwrap(); + lock_candidates(&tx, &second, "second").await.unwrap(); + tx.rollback().await.unwrap(); + store.resume.notify_one(); + delayed.await.unwrap().unwrap(); + assert!(inner.contains(&space, &value, 1).await.unwrap()); + assert_eq!( + ledger::Entity::find_by_id((SpaceIdWrap(space.clone()), value)) + .one(&fixture.conn) + .await + .unwrap() + .unwrap() + .generation, + 1 + ); + let tx = restarted.begin().await.unwrap(); + lock_candidates(&tx, &second, "second").await.unwrap(); + tx.rollback().await.unwrap(); + restarted.close().await.unwrap(); + } + + #[tokio::test] + async fn mysql_attempt_fence_preserves_other_writer_and_recovers_lost_delete() { + let Ok(url) = std::env::var("TINYCLOUD_TEST_MYSQL_URL") else { + eprintln!("skipping MySQL orphan fence (TINYCLOUD_TEST_MYSQL_URL unset)"); + return; + }; + let (prefix, _) = url.rsplit_once('/').expect("MySQL URL needs a database"); + let name = format!( + "tc735_orphan_{}_{}", + std::process::id(), + OffsetDateTime::now_utc().unix_timestamp_nanos() + ); + let admin = Database::connect(url.as_str()).await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::MySql, + format!("CREATE DATABASE {name}"), + )) + .await + .unwrap(); + let db = Database::connect(format!("{prefix}/{name}")).await.unwrap(); + // The older full MySQL migrator fails on unrelated indexed-column + // widths. Exercise the production model columns and transaction + // semantics here without altering unrelated migrations. + for ddl in [ + "CREATE TABLE kv_blob_generation ( + space VARCHAR(191) NOT NULL, value VARBINARY(34) NOT NULL, + generation BIGINT NOT NULL, PRIMARY KEY (space, value))", + "CREATE TABLE kv_orphan_candidate ( + space VARCHAR(191) NOT NULL, value VARBINARY(34) NOT NULL, + generation BIGINT NOT NULL, created_at DATETIME(6) NOT NULL, + deleting_at DATETIME(6) NULL, deletion_bytes BIGINT NULL, + PRIMARY KEY (space, value))", + "CREATE TABLE kv_orphan_attempt ( + space VARCHAR(191) NOT NULL, value VARBINARY(34) NOT NULL, + attempt_id VARCHAR(32) NOT NULL, generation BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL, PRIMARY KEY (space, value, attempt_id))", + "CREATE TABLE kv_write (space VARCHAR(191), value VARBINARY(34), + generation BIGINT NOT NULL DEFAULT 0, seq BIGINT NOT NULL, INDEX (space, value))", + "CREATE TABLE current_kv (space VARCHAR(191), value VARBINARY(34), + generation BIGINT NOT NULL DEFAULT 0, seq BIGINT NOT NULL, INDEX (space, value))", + ] { + db.execute(Statement::from_string(DbBackend::MySql, ddl)) + .await + .unwrap(); + } + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let value = upload(&inner, &space, b"mysql-overlap").await; + let blobs = [(space.clone(), value)]; + register(&db, &blobs, "a").await; + register(&db, &blobs, "b").await; + refund_failed(&db, &store, &gen0(&blobs), "a") + .await + .unwrap(); + assert!(inner.contains(&space, &value, 0).await.unwrap()); + let tx = db.begin().await.unwrap(); + lock_candidates(&tx, &gen0(&blobs), "b").await.unwrap(); + tx.rollback().await.unwrap(); + store.lose_response.store(true, Ordering::SeqCst); + assert!(refund_failed(&db, &store, &gen0(&blobs), "b") + .await + .is_err()); + assert_eq!(deleting_fences(&db).await.unwrap(), gen0(&blobs)); + let tx = db.begin().await.unwrap(); + assert!(is_candidate_unavailable( + ®ister_candidates(&tx, &blobs, "c").await.unwrap_err(), + )); + tx.rollback().await.unwrap(); + assert_eq!( + collect_candidates(&db, &store, 10, Duration::ZERO) + .await + .unwrap(), + 0 + ); + let third = register(&db, &blobs, "c").await; + assert_eq!(third[0].2, 1); + assert_eq!( + upload_generation(&inner, &space, b"mysql-overlap", 1).await, + value + ); + refund_failed(&db, &store, &third, "c").await.unwrap(); + assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + db.close().await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::MySql, + format!("DROP DATABASE {name}"), + )) + .await + .unwrap(); + admin.close().await.unwrap(); + } + #[tokio::test] async fn successful_commit_removes_candidate_without_collecting_blob() { let db = database().await; let store = MemoryStore::default(); let space = space(); let value = hash(b"committed"); - register_candidates(&db, &[(space.clone(), value)]) - .await - .unwrap(); + register(&db, &[(space.clone(), value)], "a").await; let tx = db.begin().await.unwrap(); - lock_candidates(&tx, &[(space.clone(), value)]) + lock_candidates(&tx, &[(space.clone(), value, 0)], "a") .await .unwrap(); assert_eq!(upload(&store, &space, b"committed").await, value); - finish_candidates(&tx, &[(space.clone(), value)]) + reference(&tx, "kv_write", &space, value, 0).await; + finish_candidates(&tx, &[(space.clone(), value, 0)], "a") .await .unwrap(); tx.commit().await.unwrap(); @@ -614,6 +1852,70 @@ mod tests { .unwrap(), 0 ); - assert!(store.contains(&space, &value).await.unwrap()); + assert!(store.contains(&space, &value, 0).await.unwrap()); + } + + #[tokio::test] + async fn failed_old_delete_does_not_starve_next_aged_attempt() { + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let old = upload(&inner, &space, b"old failed delete").await; + let next = upload(&inner, &space, b"next candidate").await; + register(&db, &[(space.clone(), old)], "a").await; + register(&db, &[(space.clone(), next)], "b").await; + store.lose_response.store(true, Ordering::SeqCst); + assert!(refund_failed(&db, &store, &[(space.clone(), old, 0)], "a") + .await + .is_err()); + kv_orphan_candidate::Entity::update_many() + .col_expr( + kv_orphan_candidate::Column::DeletingAt, + Expr::value(OffsetDateTime::now_utc() - Duration::hours(2)), + ) + .filter(kv_orphan_candidate::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(kv_orphan_candidate::Column::Value.eq(old)) + .exec(&db) + .await + .unwrap(); + age(&db, &space, next).await; + store.lose_response.store(true, Ordering::SeqCst); + assert!(collect_candidates(&db, &store, 1, Duration::hours(1)) + .await + .is_err()); + assert_eq!( + collect_candidates(&db, &store, 1, Duration::hours(1)) + .await + .unwrap(), + 1 + ); + assert!(!inner.contains(&space, &next, 0).await.unwrap()); + } + + #[tokio::test] + async fn failed_old_metadata_read_does_not_starve_next_attempt() { + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let old = upload(&inner, &space, b"old failed HEAD").await; + let next = upload(&inner, &space, b"next after HEAD").await; + register(&db, &[(space.clone(), old)], "a").await; + register(&db, &[(space.clone(), next)], "b").await; + age(&db, &space, old).await; + age(&db, &space, next).await; + store.fail_read.store(true, Ordering::SeqCst); + assert!(collect_candidates(&db, &store, 1, Duration::hours(1)) + .await + .is_err()); + assert!(inner.contains(&space, &old, 0).await.unwrap()); + assert_eq!( + collect_candidates(&db, &store, 1, Duration::hours(1)) + .await + .unwrap(), + 1 + ); + assert!(!inner.contains(&space, &next, 0).await.unwrap()); } } diff --git a/tinycloud-core/src/kv_sync.rs b/tinycloud-core/src/kv_sync.rs index 0a49c542..cc09a62f 100644 --- a/tinycloud-core/src/kv_sync.rs +++ b/tinycloud-core/src/kv_sync.rs @@ -895,7 +895,7 @@ pub(crate) mod tests { use crate::storage::memory::{MemoryStaging, MemoryStore}; use crate::storage::HashBuffer; use crate::InvocationOutcome; - use futures::io::AsyncWriteExt; + use futures::io::{AsyncReadExt, AsyncWriteExt}; use sea_orm::{ ActiveModelTrait, ActiveValue::Set, ConnectOptions, Database, DatabaseConnection, TransactionTrait, @@ -1680,13 +1680,104 @@ pub(crate) mod tests { ); assert!( !store - .contains(&other.space, &crate::hash::hash(b"unauthorized")) + .contains(&other.space, &crate::hash::hash(b"unauthorized"), 0) .await .unwrap(), "authorization failure must not create a blob" ); } + #[tokio::test] + async fn gen_zero_legacy_read_and_reuploaded_generation_get_and_range() { + use crate::storage::{ByteRangeSpec, ImmutableReadStore, ImmutableWriteStore, RangeRead}; + let store = MemoryStore::default(); + let db = SpaceDatabase::new( + Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(), + store.clone(), + StaticSecret::new([0u8; 32].to_vec()).unwrap(), + ) + .await + .unwrap(); + let owner = Owner::generate(); + host(&db, &owner).await; + + let original = run(&db, &owner, &[Op::Put("notes/legacy", b"bare")]).await[0]; + let legacy_key: Path = "notes/legacy".parse().unwrap(); + let legacy = current_kv::Entity::find_by_id(( + SpaceIdWrap(owner.space.clone()), + legacy_key.clone().into(), + )) + .one(db.connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(legacy.generation, 0); + assert_eq!(legacy.value, original); + assert_eq!( + store.read_to_vec(&owner.space, &original, 0).await.unwrap(), + Some(b"bare".to_vec()), + "the older bare-key reader can still read a first-generation row" + ); + + let bytes = b"regenerated"; + let hash = crate::hash::hash(bytes); + let tx = db.connection().begin().await.unwrap(); + let old = crate::kv_orphans::register_candidates( + &tx, + &[(owner.space.clone(), hash)], + "orphan-before-resave", + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!(old[0].2, 0); + let mut staged = HashBuffer::new(Vec::new()); + staged.write_all(bytes).await.unwrap(); + store.persist(&owner.space, staged, 0).await.unwrap(); + crate::kv_orphans::refund_failed(db.connection(), &store, &old, "orphan-before-resave") + .await + .unwrap(); + + assert_eq!( + run(&db, &owner, &[Op::Put("notes/resaved", bytes)]).await[0], + hash + ); + let new_key: Path = "notes/resaved".parse().unwrap(); + let new_row = current_kv::Entity::find_by_id(( + SpaceIdWrap(owner.space.clone()), + new_key.clone().into(), + )) + .one(db.connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(new_row.generation, 1); + assert!(!store.contains(&owner.space, &hash, 0).await.unwrap()); + let (_, _, content) = db.kv_get(&owner.space, &new_key).await.unwrap().unwrap(); + let (_, mut reader) = content.into_inner(); + let mut read = Vec::new(); + reader.read_to_end(&mut read).await.unwrap(); + assert_eq!(read, bytes); + let (_, _, range) = db + .kv_get_range( + &owner.space, + &new_key, + ByteRangeSpec::Inclusive { start: 2, end: 5 }, + ) + .await + .unwrap() + .unwrap(); + let RangeRead::Content { content, .. } = range else { + panic!("expected generated range"); + }; + let (_, mut reader) = content.into_inner(); + let mut read = Vec::new(); + reader.read_to_end(&mut read).await.unwrap(); + assert_eq!(read, &bytes[2..=5]); + } + #[tokio::test] async fn rejected_put_refunds_quota_without_deleting_committed_hash() { use crate::storage::{ImmutableReadStore, StoreSize}; @@ -1714,10 +1805,10 @@ pub(crate) mod tests { assert!(rejected.contains("precondition"), "{rejected}"); let orphan = crate::hash::hash(b"rejected"); assert!( - !store.contains(&owner.space, &orphan).await.unwrap(), + !store.contains(&owner.space, &orphan, 0).await.unwrap(), "a known precondition failure must remove its unreferenced upload" ); - assert!(store.contains(&owner.space, &committed).await.unwrap()); + assert!(store.contains(&owner.space, &committed, 0).await.unwrap()); assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(4)); let duplicate = invoke(&db, &owner, &[Op::Put("notes/a", b"kept")], { let mut options = KvInvokeOptions::default(); @@ -1730,7 +1821,7 @@ pub(crate) mod tests { .await .unwrap_err(); assert!(duplicate.contains("precondition"), "{duplicate}"); - assert!(store.contains(&owner.space, &committed).await.unwrap()); + assert!(store.contains(&owner.space, &committed, 0).await.unwrap()); assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(4)); } @@ -1767,7 +1858,7 @@ pub(crate) mod tests { .unwrap_err(); assert!(error.contains("blocked"), "{error}"); assert!(!store - .contains(&owner.space, &crate::hash::hash(b"uploaded")) + .contains(&owner.space, &crate::hash::hash(b"uploaded"), 0) .await .unwrap()); assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); @@ -1790,23 +1881,30 @@ pub(crate) mod tests { impl crate::storage::ImmutableReadStore for AmbiguousUploadStore { type Error = std::io::Error; type Readable = futures::io::Cursor>; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { - self.0.contains(space, id).await + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { + self.0.contains(space, id, generation).await } async fn read( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error> { - self.0.read(space, id).await + self.0.read(space, id, generation).await } async fn read_range( &self, space: &SpaceId, id: &Hash, + generation: i64, range: crate::storage::ByteRangeSpec, ) -> Result>, Self::Error> { - self.0.read_range(space, id, range).await + self.0.read_range(space, id, generation, range).await } } @@ -1817,8 +1915,9 @@ pub(crate) mod tests { &self, space: &SpaceId, staged: HashBuffer>, + generation: i64, ) -> Result { - self.0.persist(space, staged).await?; + self.0.persist(space, staged, generation).await?; Err(std::io::Error::other("lost PUT response")) } } @@ -1826,8 +1925,13 @@ pub(crate) mod tests { #[async_trait::async_trait] impl crate::storage::ImmutableDeleteStore for AmbiguousUploadStore { type Error = std::io::Error; - async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error> { - self.0.remove(space, id).await + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { + self.0.remove(space, id, generation).await } } @@ -1871,7 +1975,7 @@ pub(crate) mod tests { }; assert!(error.contains("lost PUT response"), "{error}"); assert!(!store - .contains(&owner.space, &crate::hash::hash(b"uploaded")) + .contains(&owner.space, &crate::hash::hash(b"uploaded"), 0) .await .unwrap()); assert_eq!(store.0.total_size(&owner.space).await.unwrap(), Some(0)); @@ -2028,7 +2132,12 @@ pub(crate) mod tests { let hash = crate::hash::hash(b"persist-before-lock"); let visible = tokio::time::timeout(std::time::Duration::from_secs(3), async { loop { - if fixture.store.contains(&owner.space, &hash).await.unwrap() { + if fixture + .store + .contains(&owner.space, &hash, 0) + .await + .unwrap() + { break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; diff --git a/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs b/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs index 3387b81a..1104e3cb 100644 --- a/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs +++ b/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs @@ -119,6 +119,7 @@ fn latest_writes() -> SelectStatement { (write.clone(), kv_write::Column::Metadata), ]) .expr_as(Expr::exists(exact_tombstone), current_kv::Column::Deleted) + .expr_as(Expr::val(0), current_kv::Column::Generation) .from_as(kv_write::Entity, write) .cond_where(Condition::all().not().add(Expr::exists(newer_write))); query.to_owned() @@ -141,6 +142,7 @@ async fn backfill_and_validate(manager: &SchemaManager<'_>) -> Result<(), DbErr> current_kv::Column::Value, current_kv::Column::Metadata, current_kv::Column::Deleted, + current_kv::Column::Generation, ]) .select_from(expected_select.clone()) .map_err(|error| DbErr::Migration(error.to_string()))?; @@ -233,12 +235,13 @@ mod tests { epoch: epoch_id, epoch_seq: 0, value: hash(format!("value-{label}").as_bytes()), + generation: 0, metadata: Metadata(BTreeMap::from([("label".to_string(), label.to_string())])), }; - kv_write::ActiveModel::from(write.clone()) - .insert(db) - .await - .unwrap(); + let mut legacy = kv_write::ActiveModel::from(write.clone()); + // Older writers omit the field; a fresh table defaults it to zero. + legacy.generation = sea_orm::ActiveValue::NotSet; + legacy.insert(db).await.unwrap(); write } diff --git a/tinycloud-core/src/migrations/m20260725_000000_request_path_indexes.rs b/tinycloud-core/src/migrations/m20260725_000000_request_path_indexes.rs index 8d9ea32f..6e0138fc 100644 --- a/tinycloud-core/src/migrations/m20260725_000000_request_path_indexes.rs +++ b/tinycloud-core/src/migrations/m20260725_000000_request_path_indexes.rs @@ -543,6 +543,8 @@ mod tests { epoch: Set(epoch_id), epoch_seq: Set(i), value: Set(hash(format!("tc282-value-{i}").as_bytes())), + // Legacy inserts omit generation, which defaults to zero. + generation: sea_orm::ActiveValue::NotSet, metadata: Set(Metadata(BTreeMap::new())), } .insert(db) diff --git a/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs b/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs index ef0c557f..27ced062 100644 --- a/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs +++ b/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs @@ -1,7 +1,7 @@ use sea_orm::Schema; use sea_orm_migration::prelude::*; -use crate::models::kv_orphan_candidate; +use crate::models::{kv_blob_generation, kv_orphan_attempt, kv_orphan_candidate}; #[derive(DeriveMigrationName)] pub struct Migration; @@ -10,26 +10,78 @@ pub struct Migration; impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let schema = Schema::new(manager.get_database_backend()); + manager + .create_table(schema.create_table_from_entity(kv_blob_generation::Entity)) + .await?; + for table in ["kv_write", "current_kv"] { + // A fresh install builds the initial table from today's entity, + // while an upgraded schema still has the pre-generation column set. + if !manager.has_column(table, "generation").await? { + manager + .alter_table( + Table::alter() + .table(Alias::new(table)) + .add_column( + ColumnDef::new(Alias::new("generation")) + .big_integer() + .not_null() + .default(0), + ) + .to_owned(), + ) + .await?; + } + } manager .create_table(schema.create_table_from_entity(kv_orphan_candidate::Entity)) .await?; + manager + .create_table(schema.create_table_from_entity(kv_orphan_attempt::Entity)) + .await?; manager .create_index( Index::create() - .name("idx_kv_orphan_candidate_age") + .name("idx_kv_orphan_candidate_deleting") .table(kv_orphan_candidate::Entity) - .col(kv_orphan_candidate::Column::CreatedAt) + .col(kv_orphan_candidate::Column::DeletingAt) .col(kv_orphan_candidate::Column::Space) .col(kv_orphan_candidate::Column::Value) .to_owned(), ) .await?; + manager + .create_index( + Index::create() + .name("idx_kv_orphan_attempt_age") + .table(kv_orphan_attempt::Entity) + .col(kv_orphan_attempt::Column::CreatedAt) + .col(kv_orphan_attempt::Column::Space) + .col(kv_orphan_attempt::Column::Value) + .to_owned(), + ) + .await?; Ok(()) } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + for table in ["current_kv", "kv_write"] { + manager + .alter_table( + Table::alter() + .table(Alias::new(table)) + .drop_column(Alias::new("generation")) + .to_owned(), + ) + .await?; + } + manager + .drop_table(Table::drop().table(kv_orphan_attempt::Entity).to_owned()) + .await?; manager .drop_table(Table::drop().table(kv_orphan_candidate::Entity).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(kv_blob_generation::Entity).to_owned()) .await } } diff --git a/tinycloud-core/src/models/current_kv.rs b/tinycloud-core/src/models/current_kv.rs index 2c5ce082..a033744a 100644 --- a/tinycloud-core/src/models/current_kv.rs +++ b/tinycloud-core/src/models/current_kv.rs @@ -14,6 +14,8 @@ pub struct Model { pub epoch: Hash, pub epoch_seq: i64, pub value: Hash, + #[sea_orm(default_value = 0)] + pub generation: i64, pub metadata: Metadata, pub deleted: bool, } diff --git a/tinycloud-core/src/models/invocation.rs b/tinycloud-core/src/models/invocation.rs index 4602bd80..c6cdea1d 100644 --- a/tinycloud-core/src/models/invocation.rs +++ b/tinycloud-core/src/models/invocation.rs @@ -640,6 +640,7 @@ async fn save( VersionedOperation::KvWrite { key, value, + generation, metadata, space, seq, @@ -650,6 +651,7 @@ async fn save( invocation: hash, key: key.clone().into(), value: *value, + generation: *generation, space: space.clone().into(), metadata: metadata.clone(), seq: *seq, @@ -791,6 +793,7 @@ pub(crate) async fn upsert_current_kv( epoch: write.epoch, epoch_seq: write.epoch_seq, value: write.value, + generation: write.generation, metadata: write.metadata, deleted: false, }; @@ -804,6 +807,7 @@ pub(crate) async fn upsert_current_kv( const NEWER: &str = "VALUES(`seq`) > `seq` OR (VALUES(`seq`) = `seq` AND VALUES(`epoch`) > `epoch`) OR (VALUES(`seq`) = `seq` AND VALUES(`epoch`) = `epoch` AND VALUES(`epoch_seq`) > `epoch_seq`) OR (VALUES(`seq`) = `seq` AND VALUES(`epoch`) = `epoch` AND VALUES(`epoch_seq`) = `epoch_seq` AND `deleted`)"; for (column, incoming) in [ (current_kv::Column::Invocation, "invocation"), + (current_kv::Column::Generation, "generation"), (current_kv::Column::Value, "value"), (current_kv::Column::Metadata, "metadata"), (current_kv::Column::Deleted, "deleted"), @@ -826,6 +830,7 @@ pub(crate) async fn upsert_current_kv( current_kv::Column::Epoch, current_kv::Column::EpochSeq, current_kv::Column::Value, + current_kv::Column::Generation, current_kv::Column::Metadata, current_kv::Column::Deleted, ]) @@ -1081,6 +1086,7 @@ mod tests { epoch: crate::hash::hash(format!("epoch-{label}").as_bytes()), epoch_seq: 0, value: crate::hash::hash(format!("value-{label}").as_bytes()), + generation: 0, metadata: crate::types::Metadata(BTreeMap::from([( "label".to_string(), label.to_string(), diff --git a/tinycloud-core/src/models/kv_blob_generation.rs b/tinycloud-core/src/models/kv_blob_generation.rs new file mode 100644 index 00000000..b421e7db --- /dev/null +++ b/tinycloud-core/src/models/kv_blob_generation.rs @@ -0,0 +1,18 @@ +use crate::hash::Hash; +use crate::types::SpaceIdWrap; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "kv_blob_generation")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub space: SpaceIdWrap, + #[sea_orm(primary_key, auto_increment = false)] + pub value: Hash, + pub generation: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/tinycloud-core/src/models/kv_orphan_attempt.rs b/tinycloud-core/src/models/kv_orphan_attempt.rs new file mode 100644 index 00000000..e0d33b19 --- /dev/null +++ b/tinycloud-core/src/models/kv_orphan_attempt.rs @@ -0,0 +1,22 @@ +use crate::hash::Hash; +use crate::types::SpaceIdWrap; +use sea_orm::entity::prelude::*; +use time::OffsetDateTime; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "kv_orphan_attempt")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub space: SpaceIdWrap, + #[sea_orm(primary_key, auto_increment = false)] + pub value: Hash, + #[sea_orm(primary_key, auto_increment = false)] + pub attempt_id: String, + pub generation: i64, + pub created_at: OffsetDateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/tinycloud-core/src/models/kv_orphan_candidate.rs b/tinycloud-core/src/models/kv_orphan_candidate.rs index efbfdc13..92db3626 100644 --- a/tinycloud-core/src/models/kv_orphan_candidate.rs +++ b/tinycloud-core/src/models/kv_orphan_candidate.rs @@ -10,7 +10,10 @@ pub struct Model { pub space: SpaceIdWrap, #[sea_orm(primary_key, auto_increment = false)] pub value: Hash, + pub generation: i64, pub created_at: OffsetDateTime, + pub deleting_at: Option, + pub deletion_bytes: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/tinycloud-core/src/models/kv_write.rs b/tinycloud-core/src/models/kv_write.rs index c6c1bd2e..93a7ddf7 100644 --- a/tinycloud-core/src/models/kv_write.rs +++ b/tinycloud-core/src/models/kv_write.rs @@ -16,6 +16,8 @@ pub struct Model { pub epoch: Hash, pub epoch_seq: i64, pub value: Hash, + #[sea_orm(default_value = 0)] + pub generation: i64, pub metadata: Metadata, } diff --git a/tinycloud-core/src/models/mod.rs b/tinycloud-core/src/models/mod.rs index 9cdd3e48..571b508b 100644 --- a/tinycloud-core/src/models/mod.rs +++ b/tinycloud-core/src/models/mod.rs @@ -17,7 +17,9 @@ pub mod hook_delivery; pub mod hook_subscription; pub mod invocation; pub mod invocation_replay; +pub mod kv_blob_generation; pub mod kv_delete; +pub mod kv_orphan_attempt; pub mod kv_orphan_candidate; pub mod kv_write; pub mod meeting_legacy_write_guard; diff --git a/tinycloud-core/src/storage/either.rs b/tinycloud-core/src/storage/either.rs index 9d0a0475..0e2befa5 100644 --- a/tinycloud-core/src/storage/either.rs +++ b/tinycloud-core/src/storage/either.rs @@ -25,20 +25,32 @@ where { type Readable = AsyncEither; type Error = EitherError; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { match self { - Self::A(l) => l.contains(space, id).await.map_err(Self::Error::A), - Self::B(r) => r.contains(space, id).await.map_err(Self::Error::B), + Self::A(l) => l + .contains(space, id, generation) + .await + .map_err(Self::Error::A), + Self::B(r) => r + .contains(space, id, generation) + .await + .map_err(Self::Error::B), } } async fn read( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error> { match self { Self::A(l) => l - .read(space, id) + .read(space, id, generation) .await .map(|o| { o.map(|c| { @@ -48,7 +60,7 @@ where }) .map_err(Self::Error::A), Self::B(r) => r - .read(space, id) + .read(space, id, generation) .await .map(|o| { o.map(|c| { @@ -64,11 +76,12 @@ where &self, space: &SpaceId, id: &Hash, + generation: i64, range: ByteRangeSpec, ) -> Result>, Self::Error> { match self { Self::A(store) => store - .read_range(space, id, range) + .read_range(space, id, generation, range) .await .map(|read| { read.map(|read| match read { @@ -91,7 +104,7 @@ where }) .map_err(Self::Error::A), Self::B(store) => store - .read_range(space, id, range) + .read_range(space, id, generation, range) .await .map(|read| { read.map(|read| match read { @@ -154,10 +167,17 @@ where &self, space: &SpaceId, staged: HashBuffer, + generation: i64, ) -> Result { match self { - Self::A(a) => a.persist(space, staged).await.map_err(Self::Error::A), - Self::B(b) => b.persist(space, staged).await.map_err(Self::Error::B), + Self::A(a) => a + .persist(space, staged, generation) + .await + .map_err(Self::Error::A), + Self::B(b) => b + .persist(space, staged, generation) + .await + .map_err(Self::Error::B), } } } @@ -199,10 +219,21 @@ where B: ImmutableDeleteStore, { type Error = EitherError; - async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error> { + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { match self { - Self::A(l) => l.remove(space, id).await.map_err(Self::Error::A), - Self::B(r) => r.remove(space, id).await.map_err(Self::Error::B), + Self::A(l) => l + .remove(space, id, generation) + .await + .map_err(Self::Error::A), + Self::B(r) => r + .remove(space, id, generation) + .await + .map_err(Self::Error::B), } } } @@ -221,3 +252,49 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use futures::io::{AsyncReadExt, AsyncWriteExt}; + + #[tokio::test] + async fn either_dispatches_each_generation() { + let space: SpaceId = "tinycloud:key:test:either-generations".parse().unwrap(); + for store in [ + Either::A(MemoryStore::default()), + Either::B(MemoryStore::default()), + ] { + let mut old = HashBuffer::new(Vec::new()); + old.write_all(b"same bytes").await.unwrap(); + let hash = old.hash(); + store.persist(&space, old, 0).await.unwrap(); + let mut next = HashBuffer::new(Vec::new()); + next.write_all(b"same bytes").await.unwrap(); + store.persist(&space, next, 1).await.unwrap(); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), Some(())); + assert!(!store.contains(&space, &hash, 0).await.unwrap()); + assert!(store.contains(&space, &hash, 1).await.unwrap()); + assert_eq!( + store.read_to_vec(&space, &hash, 1).await.unwrap(), + Some(b"same bytes".to_vec()) + ); + let RangeRead::Content { mut content, .. } = store + .read_range( + &space, + &hash, + 1, + ByteRangeSpec::Inclusive { start: 5, end: 9 }, + ) + .await + .unwrap() + .unwrap() + else { + panic!("expected range"); + }; + let mut bytes = Vec::new(); + content.read_to_end(&mut bytes).await.unwrap(); + assert_eq!(bytes, b"bytes"); + } + } +} diff --git a/tinycloud-core/src/storage/memory.rs b/tinycloud-core/src/storage/memory.rs index f6098470..e43ed9e5 100644 --- a/tinycloud-core/src/storage/memory.rs +++ b/tinycloud-core/src/storage/memory.rs @@ -16,7 +16,7 @@ pub struct MemoryStore { spaces: Arc>>, } -type Blocks = DashMap>; +type Blocks = DashMap<(Hash, i64), Vec>; #[derive(Default, Debug, Clone, Hash, PartialEq, Eq)] pub struct MemoryStaging; @@ -65,11 +65,16 @@ impl ImmutableReadStore for MemoryStore { type Error = io::Error; type Readable = Cursor>; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { Ok(self .spaces .get(space) - .map(|o| o.contains_key(id)) + .map(|o| o.contains_key(&(*id, generation))) .unwrap_or(false)) } @@ -77,10 +82,11 @@ impl ImmutableReadStore for MemoryStore { &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error> { let start = Instant::now(); let result = match self.spaces.get(space) { - Some(o) => match o.get(id) { + Some(o) => match o.get(&(*id, generation)) { Some(data) => { let len = data.len() as u64; let reader = Cursor::new(data.clone()); @@ -102,12 +108,13 @@ impl ImmutableReadStore for MemoryStore { &self, space: &SpaceId, id: &Hash, + generation: i64, range: ByteRangeSpec, ) -> Result>, Self::Error> { let Some(space) = self.spaces.get(space) else { return Ok(None); }; - let Some(data) = space.get(id) else { + let Some(data) = space.get(&(*id, generation)) else { return Ok(None); }; let total_size = data.len() as u64; @@ -126,9 +133,10 @@ impl ImmutableReadStore for MemoryStore { &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, VecReadError> { match self.spaces.get(space) { - Some(o) => Ok(o.get(id).map(|data| data.clone())), + Some(o) => Ok(o.get(&(*id, generation)).map(|data| data.clone())), None => Ok(None), } } @@ -142,6 +150,7 @@ impl ImmutableWriteStore for MemoryStore { &self, space: &SpaceId, mut staged: HashBuffer>, + generation: i64, ) -> Result { let start = Instant::now(); let hash = staged.hash(); @@ -154,7 +163,7 @@ impl ImmutableWriteStore for MemoryStore { .or_insert_with(|| Arc::new(DashMap::new())) .clone(); - space_storage.insert(hash, data); + space_storage.insert((hash, generation), data); crate::telemetry::observe_stage( crate::telemetry::InvocationStage::BlockWrite, crate::telemetry::StageOutcome::Ok, @@ -168,6 +177,7 @@ impl ImmutableWriteStore for MemoryStore { space: &SpaceId, mut staged: HashBuffer>, hash: &Hash, + generation: i64, ) -> Result<(), KeyedWriteError> { let start = Instant::now(); if hash != &staged.hash() { @@ -187,7 +197,7 @@ impl ImmutableWriteStore for MemoryStore { .or_insert_with(|| Arc::new(DashMap::new())) .clone(); - space_storage.insert(*hash, data); + space_storage.insert((*hash, generation), data); crate::telemetry::observe_stage( crate::telemetry::InvocationStage::BlockWrite, crate::telemetry::StageOutcome::Ok, @@ -201,11 +211,16 @@ impl ImmutableWriteStore for MemoryStore { impl ImmutableDeleteStore for MemoryStore { type Error = io::Error; - async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error> { + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { Ok(self .spaces .get(space) - .and_then(|o| o.remove(id)) + .and_then(|o| o.remove(&(*id, generation))) .map(|_| ())) } } @@ -222,3 +237,48 @@ impl StoreSize for MemoryStore { })) } } + +#[cfg(test)] +mod tests { + use super::*; + use futures::io::{AsyncReadExt, AsyncWriteExt}; + + #[tokio::test] + async fn same_hash_generations_are_independent() { + let store = MemoryStore::default(); + let space: SpaceId = "tinycloud:key:test:memory-generations".parse().unwrap(); + let mut old = HashBuffer::new(Vec::new()); + old.write_all(b"same bytes").await.unwrap(); + let hash = old.hash(); + store.persist(&space, old, 0).await.unwrap(); + let mut next = HashBuffer::new(Vec::new()); + next.write_all(b"same bytes").await.unwrap(); + store.persist(&space, next, 1).await.unwrap(); + assert_eq!(store.total_size(&space).await.unwrap(), Some(20)); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), Some(())); + assert!(!store.contains(&space, &hash, 0).await.unwrap()); + assert!(store.contains(&space, &hash, 1).await.unwrap()); + assert_eq!( + store.read_to_vec(&space, &hash, 1).await.unwrap(), + Some(b"same bytes".to_vec()) + ); + let RangeRead::Content { mut content, .. } = store + .read_range( + &space, + &hash, + 1, + ByteRangeSpec::Inclusive { start: 5, end: 9 }, + ) + .await + .unwrap() + .unwrap() + else { + panic!("expected range"); + }; + let mut bytes = Vec::new(); + content.read_to_end(&mut bytes).await.unwrap(); + assert_eq!(bytes, b"bytes"); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(store.total_size(&space).await.unwrap(), Some(10)); + } +} diff --git a/tinycloud-core/src/storage/mod.rs b/tinycloud-core/src/storage/mod.rs index 9ddebac7..d29a31b1 100644 --- a/tinycloud-core/src/storage/mod.rs +++ b/tinycloud-core/src/storage/mod.rs @@ -124,28 +124,36 @@ pub enum KeyedWriteError { pub trait ImmutableReadStore: Send + Sync { type Error: StdError + Send + Sync; type Readable: futures::io::AsyncRead + Send + Sync; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result; + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result; async fn read( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error>; async fn read_range( &self, space: &SpaceId, id: &Hash, + generation: i64, range: ByteRangeSpec, ) -> Result>, Self::Error>; async fn read_to_vec( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, VecReadError> where Self::Readable: Send, { use futures::io::AsyncReadExt; - let (l, r) = match self.read(space, id).await? { + let (l, r) = match self.read(space, id, generation).await? { None => return Ok(None), Some(r) => r.into_inner(), }; @@ -179,17 +187,19 @@ where &self, space: &SpaceId, staged: HashBuffer, + generation: i64, ) -> Result; async fn persist_keyed( &self, space: &SpaceId, mut staged: HashBuffer, hash: &Hash, + generation: i64, ) -> Result<(), KeyedWriteError> { if hash != &staged.hash() { return Err(KeyedWriteError::IncorrectHash); }; - self.persist(space, staged).await?; + self.persist(space, staged, generation).await?; Ok(()) } } @@ -197,7 +207,12 @@ where #[async_trait] pub trait ImmutableDeleteStore: Send + Sync { type Error: StdError + Send + Sync; - async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error>; + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error>; } #[async_trait] @@ -213,33 +228,41 @@ where { type Error = S::Error; type Readable = S::Readable; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { - self.contains(space, id).await + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { + (**self).contains(space, id, generation).await } async fn read( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error> { - self.read(space, id).await + (**self).read(space, id, generation).await } async fn read_range( &self, space: &SpaceId, id: &Hash, + generation: i64, range: ByteRangeSpec, ) -> Result>, Self::Error> { - self.read_range(space, id, range).await + (**self).read_range(space, id, generation, range).await } async fn read_to_vec( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, VecReadError> where Self::Readable: Send, { - self.read_to_vec(space, id).await + (**self).read_to_vec(space, id, generation).await } } @@ -267,16 +290,20 @@ where &self, space: &SpaceId, staged: HashBuffer, + generation: i64, ) -> Result { - self.persist(space, staged).await + (**self).persist(space, staged, generation).await } async fn persist_keyed( &self, space: &SpaceId, staged: HashBuffer, hash: &Hash, + generation: i64, ) -> Result<(), KeyedWriteError> { - self.persist_keyed(space, staged, hash).await + (**self) + .persist_keyed(space, staged, hash, generation) + .await } } @@ -286,8 +313,13 @@ where S: ImmutableDeleteStore, { type Error = S::Error; - async fn remove(&self, space: &SpaceId, id: &Hash) -> Result, Self::Error> { - self.remove(space, id).await + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { + (**self).remove(space, id, generation).await } } diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 3cc978fd..544fd2e0 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -622,7 +622,16 @@ pub async fn app_with_control( encryption_backend, ); - let block_store = tinycloud_config.storage.blocks.open().await?; + // Migrate before opening S3: its one bucket listing both reconstructs + // SpaceSizes and records which in-flight DELETE fences were counted. + // A separate HEAD after listing could race an old DELETE and leak quota. + // TinyCloud::new sees an already-current migration history below. + tinycloud_core::db::migrate(&database_connection).await?; + let deleting_fences = tinycloud_core::kv_orphans::deleting_fences(&database_connection).await?; + let block_store: BlockStores = match &tinycloud_config.storage.blocks { + Either::A(s3) => Either::A(s3.open_with_deleting(&deleting_fences).await?), + Either::B(local) => Either::B(local.open().await?), + }; let orphan_store = block_store.clone(); let tinycloud = TinyCloud::new(database_connection, block_store, key_setup.setup(()).await?) .await? @@ -782,7 +791,7 @@ pub async fn app_with_control( _ = interval.tick() => { loop { match tinycloud_core::kv_orphans::collect_candidates_with_progress( - &conn, &store, 256, time::Duration::minutes(5), + &conn, &store, 256, tinycloud_core::kv_orphans::ORPHAN_GRACE, ).await { Ok((_, 256)) => continue, Ok(_) => break, diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index 94cf9f1d..7415d2fb 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -2200,6 +2200,16 @@ async fn invoke_impl( if let TxStoreError::KvTooManyMutations { .. } = &e { return Err(too_many_mutations().into()); } + if let TxStoreError::KvCandidateUnavailable = &e { + // The attempted blob cannot be committed: its durable + // registration expired or a prior DELETE is in flight. + // No KV write landed; retry with a fresh invocation. + return Err(( + Status::ServiceUnavailable, + format!("{UNAVAILABLE_RETRYABLE}: {e}"), + ) + .into()); + } if let TxStoreError::Tx(TxError::Db(error) | TxError::EpochInsert(error)) = &e { return Err( database_failure(error, write_may_have_applied, e.to_string()).into() @@ -9219,6 +9229,7 @@ mod tests { epoch: Set(tinycloud_core::hash::hash(b"tc908-foreign-epoch")), epoch_seq: Set(1), value: Set(tinycloud_core::hash::hash(b"foreign secret")), + generation: Set(0), metadata: Set(Metadata(Default::default())), deleted: Set(false), } diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index b05c4940..7ee28813 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -1,4 +1,4 @@ -use super::size::SpaceSizes; +use super::{size::SpaceSizes, BlobGenerationKey}; use core::pin::Pin; use futures::{ future::{Either as AsyncEither, TryFutureExt}, @@ -10,6 +10,7 @@ use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ collections::{hash_map::Entry, HashMap}, + fmt::Write as _, io::{Error as IoError, ErrorKind, SeekFrom}, path::{Path, PathBuf}, sync::Arc, @@ -33,7 +34,7 @@ pub struct FileSystemStore { sizes: SpaceSizes, // A partial filesystem write can fail after creating a file. Reserve // before writing and refund the full reservation when deleting it. - pending: Arc>>, + pending: Arc>>, } impl FileSystemStore { @@ -47,11 +48,15 @@ impl FileSystemStore { }) } - fn get_path(&self, space: &SpaceId, mh: &Hash) -> PathBuf { + fn get_path(&self, space: &SpaceId, mh: &Hash, generation: i64) -> PathBuf { + let mut name = base64::encode_config(mh.as_ref(), base64::URL_SAFE); + if generation != 0 { + write!(&mut name, "@{generation}").expect("writing to String cannot fail"); + } self.path .join(space.suffix()) .join(space.name().as_str()) - .join(base64::encode_config(mh.as_ref(), base64::URL_SAFE)) + .join(name) } async fn increment_size(&self, space: &SpaceId, size: u64) { @@ -61,20 +66,28 @@ impl FileSystemStore { self.sizes.decrement_size(space, size).await; } - async fn reserve(&self, space: &SpaceId, hash: Hash, size: u64) { + async fn reserve(&self, space: &SpaceId, hash: Hash, generation: i64, size: u64) { let mut pending = self.pending.lock().await; - if let Entry::Vacant(slot) = pending.entry((space.clone(), hash)) { + if let Entry::Vacant(slot) = pending.entry((space.clone(), hash, generation)) { slot.insert(size); self.increment_size(space, size).await; } } - async fn settle(&self, space: &SpaceId, hash: Hash) { - self.pending.lock().await.remove(&(space.clone(), hash)); + async fn settle(&self, space: &SpaceId, hash: Hash, generation: i64) { + self.pending + .lock() + .await + .remove(&(space.clone(), hash, generation)); } - async fn refund_missing_pending(&self, space: &SpaceId, hash: Hash) { - if let Some(size) = self.pending.lock().await.remove(&(space.clone(), hash)) { + async fn refund_missing_pending(&self, space: &SpaceId, hash: Hash, generation: i64) { + if let Some(size) = self + .pending + .lock() + .await + .remove(&(space.clone(), hash, generation)) + { self.decrement_size(space, size).await; } } @@ -83,17 +96,18 @@ impl FileSystemStore { &self, space: &SpaceId, hash: Hash, + generation: i64, file: TempFileStage, ) -> Result<(), FileSystemStoreError> { - if self.contains(space, &hash).await? { - self.settle(space, hash).await; + if self.contains(space, &hash, generation).await? { + self.settle(space, hash, generation).await; return Ok(()); } let size = file.size().await?; - self.reserve(space, hash, size).await; + self.reserve(space, hash, generation, size).await; let (_, path) = file.into_inner(); - path.persist(self.get_path(space, &hash))?; - self.settle(space, hash).await; + path.persist(self.get_path(space, &hash, generation))?; + self.settle(space, hash, generation).await; Ok(()) } @@ -101,18 +115,20 @@ impl FileSystemStore { &self, space: &SpaceId, hash: Hash, + generation: i64, bytes: Vec, ) -> Result<(), FileSystemStoreError> { - if self.contains(space, &hash).await? { - self.settle(space, hash).await; + if self.contains(space, &hash, generation).await? { + self.settle(space, hash, generation).await; return Ok(()); } - self.reserve(space, hash, bytes.len() as u64).await; - let file = File::create(self.get_path(space, &hash)).await?; + self.reserve(space, hash, generation, bytes.len() as u64) + .await; + let file = File::create(self.get_path(space, &hash, generation)).await?; let mut writer = futures::io::BufWriter::new(file.compat()); writer.write_all(&bytes).await?; writer.flush().await?; - self.settle(space, hash).await; + self.settle(space, hash, generation).await; Ok(()) } } @@ -177,16 +193,22 @@ pub enum FileSystemStoreError { impl ImmutableReadStore for FileSystemStore { type Error = FileSystemStoreError; type Readable = Compat>; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { - Ok(self.get_path(space, id).exists()) + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { + Ok(self.get_path(space, id, generation).exists()) } async fn read( &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error> { let start = Instant::now(); - let result = match File::open(self.get_path(space, id)).await { + let result = match File::open(self.get_path(space, id, generation)).await { Ok(file) => { let size = file.metadata().await?.len(); Ok(Some(Content::new(size, file.take(size).compat()))) @@ -206,9 +228,10 @@ impl ImmutableReadStore for FileSystemStore { &self, space: &SpaceId, id: &Hash, + generation: i64, range: ByteRangeSpec, ) -> Result>, Self::Error> { - match File::open(self.get_path(space, id)).await { + match File::open(self.get_path(space, id, generation)).await { Ok(mut file) => { let total_size = file.metadata().await?.len(); let Some(range) = range.resolve(total_size) else { @@ -336,12 +359,13 @@ impl ImmutableWriteStore for FileSystemStore { &self, space: &SpaceId, staged: HashBuffer<::Writable>, + generation: i64, ) -> Result { let start = Instant::now(); let result = async move { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - self.persist_temp(space, hash, f).await?; + self.persist_temp(space, hash, generation, f).await?; Ok(hash) } .await; @@ -361,12 +385,13 @@ impl ImmutableWriteStore for FileSystemStore { &self, space: &SpaceId, staged: HashBuffer<::Writable>, + generation: i64, ) -> Result { let start = Instant::now(); let result = async move { let (mut h, v) = staged.into_inner(); let hash = h.finalize(); - self.persist_memory(space, hash, v).await?; + self.persist_memory(space, hash, generation, v).await?; Ok(hash) } .await; @@ -388,6 +413,7 @@ impl ImmutableWriteStore as ImmutableStaging>::Writable>, + generation: i64, ) -> Result { let start = Instant::now(); let result = async move { @@ -395,8 +421,10 @@ impl ImmutableWriteStore self.persist_temp(space, hash, file).await?, - AsyncEither::Right(bytes) => self.persist_memory(space, hash, bytes).await?, + AsyncEither::Left(file) => self.persist_temp(space, hash, generation, file).await?, + AsyncEither::Right(bytes) => { + self.persist_memory(space, hash, generation, bytes).await? + } } Ok(hash) } @@ -413,12 +441,17 @@ impl ImmutableWriteStore Result, Self::Error> { - let path = self.get_path(space, id); + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { + let path = self.get_path(space, id, generation); let size = match metadata(&path).await { Ok(m) => m.len(), Err(e) if e.kind() == ErrorKind::NotFound => { - self.refund_missing_pending(space, *id).await; + self.refund_missing_pending(space, *id, generation).await; return Ok(None); } Err(e) => return Err(e.into()), @@ -429,13 +462,13 @@ impl ImmutableDeleteStore for FileSystemStore { .pending .lock() .await - .remove(&(space.clone(), *id)) + .remove(&(space.clone(), *id, generation)) .unwrap_or(size); self.decrement_size(space, charged).await; Ok(Some(())) } Err(e) if e.kind() == ErrorKind::NotFound => { - self.refund_missing_pending(space, *id).await; + self.refund_missing_pending(space, *id, generation).await; Ok(None) } Err(e) => Err(e.into()), @@ -469,13 +502,13 @@ mod test { stage.write_all(b"failed-write").await.unwrap(); let hash = stage.hash(); assert!( - ImmutableWriteStore::::persist(&store, &space, stage) + ImmutableWriteStore::::persist(&store, &space, stage, 0) .await .is_err() ); assert_eq!(store.total_size(&space).await.unwrap(), Some(12)); - assert_eq!(store.remove(&space, &hash).await.unwrap(), None); - assert!(!store.contains(&space, &hash).await.unwrap()); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert!(!store.contains(&space, &hash, 0).await.unwrap()); assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); } @@ -493,11 +526,11 @@ mod test { let mut stage = tfs.stage(&space_id).await.unwrap(); futures::io::copy(&mut &data[..], &mut stage).await.unwrap(); - let hash = ImmutableWriteStore::::persist(&store, &space_id, stage) + let hash = ImmutableWriteStore::::persist(&store, &space_id, stage, 0) .await .unwrap(); - assert!(store.contains(&space_id, &hash).await.unwrap()); + assert!(store.contains(&space_id, &hash, 0).await.unwrap()); assert_eq!( store.total_size(&space_id).await.unwrap(), Some(data.len() as u64) @@ -505,7 +538,7 @@ mod test { let mut buf = Vec::new(); store - .read(&space_id, &hash) + .read(&space_id, &hash, 0) .await .unwrap() .unwrap() @@ -515,7 +548,11 @@ mod test { assert_eq!(buf, data); assert_eq!( - store.read_to_vec(&space_id, &hash).await.unwrap().unwrap(), + store + .read_to_vec(&space_id, &hash, 0) + .await + .unwrap() + .unwrap(), data ); @@ -523,6 +560,7 @@ mod test { .read_range( &space_id, &hash, + 0, ByteRangeSpec::Inclusive { start: 6, end: 10 }, ) .await @@ -549,20 +587,98 @@ mod test { assert!(matches!( store - .read_range(&space_id, &hash, ByteRangeSpec::From { start: 99 }) + .read_range(&space_id, &hash, 0, ByteRangeSpec::From { start: 99 }) .await .unwrap() .unwrap(), RangeRead::Unsatisfiable { total_size: 11 } )); - assert_eq!(store.remove(&space_id, &hash).await.unwrap(), Some(())); - assert_eq!(store.remove(&space_id, &hash).await.unwrap(), None); - assert!(!store.contains(&space_id, &hash).await.unwrap()); + assert_eq!(store.remove(&space_id, &hash, 0).await.unwrap(), Some(())); + assert_eq!(store.remove(&space_id, &hash, 0).await.unwrap(), None); + assert!(!store.contains(&space_id, &hash, 0).await.unwrap()); assert_eq!(store.total_size(&space_id).await.unwrap(), Some(0)); assert_eq!( - store.read(&space_id, &hash).await.unwrap().map(|_| ()), + store.read(&space_id, &hash, 0).await.unwrap().map(|_| ()), None ); } + #[tokio::test] + async fn reads_preexisting_bare_hash_file_as_generation_zero() { + let dir = tempfile::tempdir().unwrap(); + let space: SpaceId = "tinycloud:key:test:old-file".parse().unwrap(); + let initial = FileSystemConfig::new(dir.path()).open().await.unwrap(); + initial.create(&space).await.unwrap(); + let mut data = HashBuffer::new(Vec::new()); + data.write_all(b"legacy").await.unwrap(); + let hash = data.hash(); + let bare = initial.get_path(&space, &hash, 0); + tokio::fs::write(&bare, b"legacy").await.unwrap(); + let loaded = FileSystemConfig::new(dir.path()).open().await.unwrap(); + assert_eq!( + loaded.read_to_vec(&space, &hash, 0).await.unwrap(), + Some(b"legacy".to_vec()) + ); + assert_eq!(loaded.read_to_vec(&space, &hash, 1).await.unwrap(), None); + assert_eq!(loaded.total_size(&space).await.unwrap(), Some(6)); + } + + #[tokio::test] + async fn generations_keep_legacy_key_and_account_for_both_files_on_restart() { + let dir = tempfile::tempdir().unwrap(); + let space: SpaceId = "tinycloud:key:test:generations".parse().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + store.create(&space).await.unwrap(); + let mut old = HashBuffer::new(Vec::new()); + old.write_all(b"same bytes").await.unwrap(); + let hash = old.hash(); + let legacy_path = store.get_path(&space, &hash, 0); + assert_eq!( + legacy_path.file_name().unwrap(), + base64::encode_config(hash.as_ref(), base64::URL_SAFE).as_str() + ); + ImmutableWriteStore::::persist(&store, &space, old, 0) + .await + .unwrap(); + assert_eq!( + store.read_to_vec(&space, &hash, 0).await.unwrap().unwrap(), + b"same bytes" + ); + let mut next = HashBuffer::new(Vec::new()); + next.write_all(b"same bytes").await.unwrap(); + ImmutableWriteStore::::persist(&store, &space, next, 1) + .await + .unwrap(); + assert_ne!(legacy_path, store.get_path(&space, &hash, 1)); + assert_eq!(store.total_size(&space).await.unwrap(), Some(20)); + let restarted = FileSystemConfig::new(dir.path()).open().await.unwrap(); + assert_eq!(restarted.total_size(&space).await.unwrap(), Some(20)); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), Some(())); + assert!(store.contains(&space, &hash, 1).await.unwrap()); + assert_eq!( + store.read_to_vec(&space, &hash, 1).await.unwrap().unwrap(), + b"same bytes" + ); + let RangeRead::Content { mut content, .. } = store + .read_range( + &space, + &hash, + 1, + ByteRangeSpec::Inclusive { start: 5, end: 9 }, + ) + .await + .unwrap() + .unwrap() + else { + panic!("expected generation-one range"); + }; + let mut range_bytes = Vec::new(); + content.read_to_end(&mut range_bytes).await.unwrap(); + assert_eq!(range_bytes, b"bytes"); + assert_eq!(store.total_size(&space).await.unwrap(), Some(10)); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(store.total_size(&space).await.unwrap(), Some(10)); + assert_eq!(store.remove(&space, &hash, 1).await.unwrap(), Some(())); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } } diff --git a/tinycloud-node-server/src/storage/mod.rs b/tinycloud-node-server/src/storage/mod.rs index df299470..b665d262 100644 --- a/tinycloud-node-server/src/storage/mod.rs +++ b/tinycloud-node-server/src/storage/mod.rs @@ -1,3 +1,10 @@ pub mod file_system; pub mod s3; pub mod size; + +use tinycloud_auth::resource::SpaceId; +use tinycloud_core::hash::Hash; + +/// One physical blob lifetime; a later generation must never share its +/// quota reservation or deletion target with an earlier generation. +type BlobGenerationKey = (SpaceId, Hash, i64); diff --git a/tinycloud-node-server/src/storage/s3.rs b/tinycloud-node-server/src/storage/s3.rs index 9322743c..f20ecad2 100644 --- a/tinycloud-node-server/src/storage/s3.rs +++ b/tinycloud-node-server/src/storage/s3.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use std::{ collections::{hash_map::Entry, HashMap}, + fmt::Write as _, future::Future, io::Error as IoError, ops::AddAssign, @@ -24,7 +25,7 @@ use std::{ use tinycloud_auth::resource::SpaceId; use tinycloud_core::{hash::Hash, storage::*}; -use super::{file_system, size::SpaceSizes}; +use super::{file_system, size::SpaceSizes, BlobGenerationKey}; use tokio::sync::Mutex; async fn aws_config() -> SdkConfig { @@ -39,7 +40,11 @@ pub struct S3BlockStore { // A PUT can succeed on S3 even if its response fails. Reserve its size // before sending; this shared map refunds a missing object on known // failure without subtracting another writer's committed bytes. - pending: Arc>>, + pending: Arc>>, + // A DELETE can land on S3 while its response is lost. Keep the pre-DELETE + // size until absence is confirmed, so retry refunds exactly once. On + // restart, the initial bucket listing reconstructs only physical bytes. + deleting: Arc>>, } #[serde_as] @@ -55,7 +60,18 @@ pub struct S3BlockConfig { impl StorageConfig for S3BlockConfig { type Error = S3Error; async fn open(&self) -> Result { - S3BlockStore::new_(self).await + S3BlockStore::new_(self, &[]).await + } +} + +impl S3BlockConfig { + /// Reconstruct quota and the charge for in-progress DELETEs from one + /// bucket listing, before the node starts its orphan collector. + pub async fn open_with_deleting( + &self, + deleting: &[(SpaceId, Hash, i64)], + ) -> Result { + S3BlockStore::new_(self, deleting).await } } @@ -79,48 +95,84 @@ async fn new_client(config: &S3BlockConfig) -> Client { Client::from_conf(sdk_config) } +async fn file_body( + file: file_system::TempFileStage, +) -> Result<(u64, ByteStream, tempfile::TempPath), S3StoreError> { + let size = file.size().await?; + let (_file, path) = file.into_inner(); + let body = ByteStream::from_path(&path).await?; + Ok((size, body, path)) +} + impl S3BlockStore { - async fn new_(config: &S3BlockConfig) -> Result { + async fn new_( + config: &S3BlockConfig, + deleting: &[(SpaceId, Hash, i64)], + ) -> Result { let client = new_client(config).await; - let sizes = client + let deleting_keys: HashMap<_, _> = deleting + .iter() + .map(|(space, hash, generation)| { + ( + Self::key_for(space, hash, *generation), + (space.clone(), *hash, *generation), + ) + }) + .collect(); + let wanted = &deleting_keys; + let (sizes, counted_deletes) = client .list_objects_v2() .bucket(&config.bucket) .into_paginator() .send() - // get the sum of all objects in each page - .try_fold(HashMap::new(), |mut acc, page| async move { - // get the sum of all objects per space in this particular page - for (space, obj_size) in page.contents.into_iter().flatten().filter_map(|content| { - content.key().and_then(|key| { - let (o, _) = key.rsplit_once('/')?; - let space: SpaceId = o.parse().ok()?; - if content.size() > 0 { - Some((space, content.size() as u64)) - } else { - None + .try_fold( + (HashMap::new(), HashMap::new()), + |(mut sizes, mut counted), page| async move { + for content in page.contents.into_iter().flatten() { + let Some(key) = content.key() else { continue }; + let Some((prefix, _)) = key.rsplit_once('/') else { + continue; + }; + let Ok(space) = prefix.parse::() else { + continue; + }; + let size = content.size(); + if size > 0 { + sizes.entry(space).or_insert(0).add_assign(size as u64); } - }) - }) { - acc.entry(space).or_insert(0).add_assign(obj_size); - } - Ok(acc) - }) - .await? - .into(); + if let Some((space, hash, generation)) = wanted.get(key) { + counted.insert((space.clone(), *hash, *generation), size.max(0) as u64); + } + } + Ok((sizes, counted)) + }, + ) + .await?; + let sizes = sizes.into(); Ok(S3BlockStore { client, bucket: config.bucket.clone(), sizes, pending: Arc::new(Mutex::new(HashMap::new())), + deleting: Arc::new(Mutex::new(counted_deletes)), }) } - fn key(&self, space: &SpaceId, id: &Hash) -> String { - format!( - "{}/{}", - space, - base64::encode_config(id.as_ref(), base64::URL_SAFE) - ) + fn key_for(space: &SpaceId, id: &Hash, generation: i64) -> String { + let mut key = format!("{space}/"); + // Hash digests fit in 64 bytes; encode on the stack so generated + // reads allocate only their storage-key String. + let mut encoded = [0u8; 88]; + let len = base64::encode_config_slice(id.as_ref(), base64::URL_SAFE, &mut encoded); + key.push_str(std::str::from_utf8(&encoded[..len]).expect("base64 is ASCII")); + if generation != 0 { + write!(&mut key, "@{generation}").expect("writing to String cannot fail"); + } + key + } + + fn key(&self, space: &SpaceId, id: &Hash, generation: i64) -> String { + Self::key_for(space, id, generation) } async fn increment_size(&self, space: &SpaceId, size: u64) { @@ -130,44 +182,67 @@ impl S3BlockStore { self.sizes.decrement_size(space, size).await; } - async fn put_if_absent( + async fn put_if_absent( &self, space: &SpaceId, hash: Hash, + generation: i64, body: F, ) -> Result<(), S3StoreError> where F: FnOnce() -> Fut, - Fut: Future>, + Fut: Future>, { - let key = (space.clone(), hash); - if self.contains(space, &hash).await? { + let key = (space.clone(), hash, generation); + if self.contains(space, &hash, generation).await? { // A retry of an ambiguous PUT can find its original object. self.pending.lock().await.remove(&key); return Ok(()); } - let (size, body) = body().await?; + let (size, body, guard) = body().await?; let mut pending = self.pending.lock().await; if let Entry::Vacant(slot) = pending.entry(key.clone()) { slot.insert(size); self.increment_size(space, size).await; } drop(pending); - self.client + let upload = self + .client .put_object() .bucket(&self.bucket) - .key(self.key(space, &hash)) + .key(self.key(space, &hash, generation)) .body(body) .send() - .await - .map_err(S3Error::from)?; + .await; + drop(guard); + upload.map_err(S3Error::from)?; self.pending.lock().await.remove(&key); Ok(()) } - async fn refund_missing_pending(&self, space: &SpaceId, id: &Hash) { - if let Some(size) = self.pending.lock().await.remove(&(space.clone(), *id)) { + async fn refund_missing_pending(&self, space: &SpaceId, id: &Hash, generation: i64) { + let key = (space.clone(), *id, generation); + let mut pending = self.pending.lock().await; + if let Some(size) = pending.get(&key).copied() { self.decrement_size(space, size).await; + pending.remove(&key); + } + } + + async fn refund_confirmed_absence(&self, space: &SpaceId, id: &Hash, generation: i64) { + let key = (space.clone(), *id, generation); + let mut deleting = self.deleting.lock().await; + if let Some(size) = deleting.get(&key).copied() { + let mut pending = self.pending.lock().await; + // Hold both reservations until the quota update finishes. Once + // it runs, both removals are synchronous and cancellation cannot + // cause a second refund on the next HEAD 404. + self.decrement_size(space, size).await; + deleting.remove(&key); + pending.remove(&key); + } else { + drop(deleting); + self.refund_missing_pending(space, id, generation).await; } } } @@ -194,12 +269,17 @@ pub enum S3StoreError { impl ImmutableReadStore for S3BlockStore { type Error = S3StoreError; type Readable = IntoAsyncRead IoError>>; - async fn contains(&self, space: &SpaceId, id: &Hash) -> Result { + async fn contains( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result { match self .client .head_object() .bucket(&self.bucket) - .key(self.key(space, id)) + .key(self.key(space, id, generation)) .send() .await { @@ -220,13 +300,14 @@ impl ImmutableReadStore for S3BlockStore { &self, space: &SpaceId, id: &Hash, + generation: i64, ) -> Result>, Self::Error> { let start = Instant::now(); let res = self .client .get_object() .bucket(&self.bucket) - .key(self.key(space, id)) + .key(self.key(space, id, generation)) .send() .await; let result = match res { @@ -258,9 +339,10 @@ impl ImmutableReadStore for S3BlockStore { &self, space: &SpaceId, id: &Hash, + generation: i64, range: ByteRangeSpec, ) -> Result>, Self::Error> { - let key = self.key(space, id); + let key = self.key(space, id, generation); let output = self .client .get_object() @@ -301,7 +383,7 @@ impl ImmutableReadStore for S3BlockStore { .client .head_object() .bucket(&self.bucket) - .key(self.key(space, id)) + .key(self.key(space, id, generation)) .send() .await; match head { @@ -357,14 +439,15 @@ impl ImmutableWriteStore for S3BlockStore { &self, space: &SpaceId, staged: HashBuffer<::Writable>, + generation: i64, ) -> Result { let start = Instant::now(); let result = async move { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - self.put_if_absent(space, hash, || async move { - Ok((f.len() as u64, ByteStream::from(f))) + self.put_if_absent(space, hash, generation, || async move { + Ok((f.len() as u64, ByteStream::from(f), ())) }) .await?; Ok(hash) @@ -386,18 +469,15 @@ impl ImmutableWriteStore for S3BlockStore { &self, space: &SpaceId, staged: HashBuffer<::Writable>, + generation: i64, ) -> Result { let start = Instant::now(); let result = async move { let (mut h, f) = staged.into_inner(); let hash = h.finalize(); - self.put_if_absent(space, hash, || async move { - let size = f.size().await?; - let (_file, path) = f.into_inner(); - Ok((size, ByteStream::from_path(&path).await?)) - }) - .await?; + self.put_if_absent(space, hash, generation, || file_body(f)) + .await?; Ok(hash) } .await; @@ -419,6 +499,7 @@ impl ImmutableWriteStore as ImmutableStaging>::Writable>, + generation: i64, ) -> Result { let start = Instant::now(); let result = async move { @@ -427,16 +508,12 @@ impl ImmutableWriteStore { - self.put_if_absent(space, hash, || async move { - let size = t_file.size().await?; - let (_file, path) = t_file.into_inner(); - Ok((size, ByteStream::from_path(&path).await?)) - }) - .await?; + self.put_if_absent(space, hash, generation, || file_body(t_file)) + .await?; } AsyncEither::Right(b) => { - self.put_if_absent(space, hash, || async move { - Ok((b.len() as u64, ByteStream::from(b))) + self.put_if_absent(space, hash, generation, || async move { + Ok((b.len() as u64, ByteStream::from(b), ())) }) .await?; } @@ -456,12 +533,17 @@ impl ImmutableWriteStore Result, Self::Error> { + async fn remove( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result, Self::Error> { let size: u64 = match self .client .head_object() .bucket(&self.bucket) - .key(self.key(space, id)) + .key(self.key(space, id, generation)) .send() .await { @@ -474,30 +556,28 @@ impl ImmutableDeleteStore for S3BlockStore { }, .. }) => { - self.refund_missing_pending(space, id).await; + self.refund_confirmed_absence(space, id, generation).await; return Ok(None); } Err(error) => return Err(S3Error::from(error).into()), }; + self.deleting + .lock() + .await + .entry((space.clone(), *id, generation)) + .or_insert(size); match self .client .delete_object() .bucket(&self.bucket) - .key(self.key(space, id)) + .key(self.key(space, id, generation)) .send() .await { Ok(_) => { - let charged = self - .pending - .lock() - .await - .remove(&(space.clone(), *id)) - .unwrap_or(size); - self.decrement_size(space, charged).await; + self.refund_confirmed_absence(space, id, generation).await; Ok(Some(())) } - // TODO does this distinguish between object missing and object present? Err(e) => Err(S3Error::from(e).into()), } } @@ -515,6 +595,419 @@ impl StoreSize for S3BlockStore { mod tests { use super::*; + #[tokio::test] + async fn filesystem_body_keeps_staged_file_until_deferred_read() { + use futures::io::AsyncWriteExt; + + let data = b"nonempty deferred S3 upload"; + let mut file = file_system::TempFileStage::new(tempfile::NamedTempFile::new().unwrap()); + file.write_all(data).await.unwrap(); + file.flush().await.unwrap(); + + let (size, body, guard) = file_body(file).await.unwrap(); + assert_eq!(size, data.len() as u64); + assert_eq!(body.collect().await.unwrap().into_bytes().as_ref(), data); + drop(guard); + } + + #[tokio::test] + async fn minio_persists_each_staging_arm_and_accepts_restage_retry() { + use futures::io::AsyncWriteExt; + + let Ok(endpoint) = std::env::var("TC735_TEST_MINIO_ENDPOINT") else { + return; + }; + let bucket = std::env::var("TC735_TEST_MINIO_BUCKET") + .expect("TC735_TEST_MINIO_BUCKET is required with TC735_TEST_MINIO_ENDPOINT"); + let store = S3BlockConfig { + bucket: bucket.clone(), + endpoint: Some(endpoint.parse().unwrap()), + } + .open() + .await + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let space: SpaceId = format!("tinycloud:key:test:tc735-stage-{nonce}") + .parse() + .unwrap(); + store.create(&space).await.unwrap(); + let mut hashes = Vec::new(); + + for mode in 0..3 { + let data = format!( + "TC-735 S3 staged upload {nonce} {} {mode}", + std::process::id() + ) + .into_bytes(); + let mut initial_hash = None; + for _ in 0..2 { + // A retry re-stages the same bytes; it never rewinds an SDK body. + let hash = if mode == 0 { + let mut stage = file_system::TempFileSystemStage + .stage(&space) + .await + .unwrap(); + stage.write_all(&data).await.unwrap(); + ImmutableWriteStore::::persist( + &store, &space, stage, 0, + ) + .await + .unwrap() + } else { + let staging = if mode == 1 { + either::Either::A(file_system::TempFileSystemStage) + } else { + either::Either::B(memory::MemoryStaging) + }; + let mut stage = staging.stage(&space).await.unwrap(); + stage.write_all(&data).await.unwrap(); + ImmutableWriteStore::< + either::Either, + >::persist(&store, &space, stage, 0) + .await + .unwrap() + }; + assert_eq!(*initial_hash.get_or_insert(hash), hash); + if !hashes.contains(&hash) { + hashes.push(hash); + } + let object = store + .client + .get_object() + .bucket(&bucket) + .key(store.key(&space, &hash, 0)) + .send() + .await + .unwrap(); + assert_eq!( + object.body.collect().await.unwrap().into_bytes().as_ref(), + data.as_slice() + ); + } + } + for hash in hashes { + store.remove(&space, &hash, 0).await.unwrap(); + } + } + + #[tokio::test] + async fn minio_lost_delete_response_refunds_once_and_restart_reconstructs() { + use futures::io::AsyncWriteExt; + + let Ok(endpoint) = std::env::var("TC735_TEST_MINIO_ENDPOINT") else { + return; + }; + let bucket = std::env::var("TC735_TEST_MINIO_BUCKET") + .expect("TC735_TEST_MINIO_BUCKET is required with TC735_TEST_MINIO_ENDPOINT"); + let config = S3BlockConfig { + bucket: bucket.clone(), + endpoint: Some(endpoint.parse().unwrap()), + }; + let store = config.open().await.unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let space: SpaceId = format!("tinycloud:key:test:tc735-delete-{nonce}") + .parse() + .unwrap(); + store.create(&space).await.unwrap(); + let data = format!("TC-735 ambiguous DELETE {} {nonce}", std::process::id()).into_bytes(); + let mut stage = memory::MemoryStaging.stage(&space).await.unwrap(); + stage.write_all(&data).await.unwrap(); + let hash = ImmutableWriteStore::::persist(&store, &space, stage, 0) + .await + .unwrap(); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(data.len() as u64) + ); + + // A replacement process lists the object *before* the old DELETE + // lands. Its deletion reservation must come from that same listing. + let counted_boot = config + .open_with_deleting(&[(space.clone(), hash, 0)]) + .await + .unwrap(); + let counted_before = counted_boot.total_size(&space).await.unwrap().unwrap(); + assert_eq!( + counted_boot + .deleting + .lock() + .await + .get(&(space.clone(), hash, 0)), + Some(&(data.len() as u64)) + ); + // DELETE lands on S3, but its response is lost. The original process + // and the boot-counted process each reconcile their own physical count. + store + .deleting + .lock() + .await + .insert((space.clone(), hash, 0), data.len() as u64); + store + .client + .delete_object() + .bucket(&bucket) + .key(store.key(&space, &hash, 0)) + .send() + .await + .unwrap(); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert_eq!(counted_boot.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!( + counted_boot.total_size(&space).await.unwrap(), + Some(counted_before - data.len() as u64) + ); + assert_eq!(counted_boot.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!( + counted_boot.total_size(&space).await.unwrap(), + Some(counted_before - data.len() as u64) + ); + + // DELETE before startup, or a crash after the first refund but before + // the DB fence was removed: neither boot listing counted this object. + let after_boot = config + .open_with_deleting(&[(space.clone(), hash, 0)]) + .await + .unwrap(); + let rebuilt = after_boot.total_size(&space).await.unwrap(); + assert!(!after_boot + .deleting + .lock() + .await + .contains_key(&(space.clone(), hash, 0))); + assert_eq!(after_boot.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(after_boot.total_size(&space).await.unwrap(), rebuilt); + } + + #[tokio::test] + async fn minio_postgres_boot_reconciles_deleting_fence_from_same_listing() { + use futures::io::AsyncWriteExt; + use tinycloud_core::{ + db::migrate, + db_pool::RecoveringConnection, + kv_orphans::{collect_candidates, deleting_fences, register_candidates}, + sea_orm::{ + ConnectOptions, ConnectionTrait, Database, DbBackend, Statement, TransactionTrait, + }, + }; + + let (Ok(endpoint), Ok(pg_url)) = ( + std::env::var("TC735_TEST_MINIO_ENDPOINT"), + std::env::var("TINYCLOUD_TEST_POSTGRES_URL"), + ) else { + return; + }; + let bucket = std::env::var("TC735_TEST_MINIO_BUCKET") + .expect("TC735_TEST_MINIO_BUCKET is required with TC735_TEST_MINIO_ENDPOINT"); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let schema = format!("tc735_s3_boot_{}_{}", std::process::id(), nonce); + let admin = Database::connect(pg_url.as_str()).await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::Postgres, + format!("CREATE SCHEMA {schema}"), + )) + .await + .unwrap(); + let mut options = ConnectOptions::new(pg_url); + options.set_schema_search_path(schema.clone()); + let conn = Database::connect(options).await.unwrap(); + migrate(&RecoveringConnection::new(conn.clone())) + .await + .unwrap(); + + let config = S3BlockConfig { + bucket: bucket.clone(), + endpoint: Some(endpoint.parse().unwrap()), + }; + let uploader = config.open().await.unwrap(); + let space: SpaceId = format!("tinycloud:key:test:tc735-boot-{nonce}") + .parse() + .unwrap(); + uploader.create(&space).await.unwrap(); + for delete_before_boot in [true, false] { + let data = format!("TC-735 PG boot {nonce} {delete_before_boot}").into_bytes(); + let mut stage = memory::MemoryStaging.stage(&space).await.unwrap(); + stage.write_all(&data).await.unwrap(); + let hash = + ImmutableWriteStore::::persist(&uploader, &space, stage, 0) + .await + .unwrap(); + let tx = conn.begin().await.unwrap(); + let registered = register_candidates(&tx, &[(space.clone(), hash)], "aborted") + .await + .unwrap(); + assert_eq!(registered, vec![(space.clone(), hash, 0)]); + tx.execute(Statement::from_sql_and_values( + DbBackend::Postgres, + "DELETE FROM kv_orphan_attempt WHERE space = $1 AND value = $2", + [space.to_string().into(), hash.into()], + )) + .await + .unwrap(); + tx.execute(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE kv_orphan_candidate SET deleting_at = CURRENT_TIMESTAMP - INTERVAL '1 second', + deletion_bytes = $3 WHERE space = $1 AND value = $2", + [ + space.to_string().into(), + hash.into(), + (data.len() as i64).into(), + ], + )) + .await + .unwrap(); + tx.commit().await.unwrap(); + let deleting = deleting_fences(&conn).await.unwrap(); + assert_eq!(deleting, vec![(space.clone(), hash, 0)]); + if delete_before_boot { + uploader + .client + .delete_object() + .bucket(&bucket) + .key(uploader.key(&space, &hash, 0)) + .send() + .await + .unwrap(); + } + let boot = config.open_with_deleting(&deleting).await.unwrap(); + let counted = boot.total_size(&space).await.unwrap().unwrap_or(0); + assert_eq!( + boot.deleting + .lock() + .await + .contains_key(&(space.clone(), hash, 0)), + !delete_before_boot, + ); + if !delete_before_boot { + assert_eq!(counted, data.len() as u64); + uploader + .client + .delete_object() + .bucket(&bucket) + .key(uploader.key(&space, &hash, 0)) + .send() + .await + .unwrap(); + } + assert_eq!( + collect_candidates(&conn, &boot, 10, time::Duration::ZERO) + .await + .unwrap(), + 0 + ); + assert!(deleting_fences(&conn).await.unwrap().is_empty()); + assert_eq!( + boot.total_size(&space).await.unwrap().unwrap_or(0), + if delete_before_boot { + counted + } else { + counted - data.len() as u64 + }, + ); + } + conn.close().await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::Postgres, + format!("DROP SCHEMA {schema} CASCADE"), + )) + .await + .unwrap(); + admin.close().await.unwrap(); + } + + #[tokio::test] + async fn minio_generations_survive_old_delete_and_rebuild_quota_together() { + use futures::io::{AsyncReadExt, AsyncWriteExt}; + let Ok(endpoint) = std::env::var("TC735_TEST_MINIO_ENDPOINT") else { + return; + }; + let bucket = std::env::var("TC735_TEST_MINIO_BUCKET") + .expect("TC735_TEST_MINIO_BUCKET is required with TC735_TEST_MINIO_ENDPOINT"); + let config = S3BlockConfig { + bucket, + endpoint: Some(endpoint.parse().unwrap()), + }; + let store = config.open().await.unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let space: SpaceId = format!("tinycloud:key:test:tc735-generation-{nonce}") + .parse() + .unwrap(); + store.create(&space).await.unwrap(); + let mut old = memory::MemoryStaging.stage(&space).await.unwrap(); + old.write_all(b"same bytes").await.unwrap(); + let hash = old.hash(); + let legacy_key = format!( + "{}/{}", + space, + base64::encode_config(hash.as_ref(), base64::URL_SAFE) + ); + assert_eq!(store.key(&space, &hash, 0), legacy_key); + ImmutableWriteStore::::persist(&store, &space, old, 0) + .await + .unwrap(); + let mut next = memory::MemoryStaging.stage(&space).await.unwrap(); + next.write_all(b"same bytes").await.unwrap(); + ImmutableWriteStore::::persist(&store, &space, next, 1) + .await + .unwrap(); + assert_eq!(store.key(&space, &hash, 1), format!("{legacy_key}@1")); + let boot = config + .open_with_deleting(&[(space.clone(), hash, 0)]) + .await + .unwrap(); + assert_eq!(boot.total_size(&space).await.unwrap(), Some(20)); + assert_eq!( + boot.deleting.lock().await.get(&(space.clone(), hash, 0)), + Some(&10) + ); + assert_eq!( + boot.read_to_vec(&space, &hash, 0).await.unwrap(), + Some(b"same bytes".to_vec()) + ); + let RangeRead::Content { mut content, .. } = boot + .read_range( + &space, + &hash, + 1, + ByteRangeSpec::Inclusive { start: 5, end: 9 }, + ) + .await + .unwrap() + .unwrap() + else { + panic!("expected generation-one range"); + }; + let mut range_bytes = Vec::new(); + content.read_to_end(&mut range_bytes).await.unwrap(); + assert_eq!(range_bytes, b"bytes"); + assert_eq!(boot.remove(&space, &hash, 0).await.unwrap(), Some(())); + assert_eq!(boot.remove(&space, &hash, 0).await.unwrap(), None); + assert!(boot.contains(&space, &hash, 1).await.unwrap()); + assert_eq!( + boot.read_to_vec(&space, &hash, 1).await.unwrap(), + Some(b"same bytes".to_vec()) + ); + assert_eq!(boot.total_size(&space).await.unwrap(), Some(10)); + boot.remove(&space, &hash, 1).await.unwrap(); + assert_eq!(boot.total_size(&space).await.unwrap(), Some(0)); + } + #[test] fn parses_s3_content_range() { let (range, total_size) = parse_content_range("bytes 1048576-2097151/67108864").unwrap(); diff --git a/tinycloud-node-server/tests/postgres_connection_recovery.rs b/tinycloud-node-server/tests/postgres_connection_recovery.rs index ca203fa9..1f150137 100644 --- a/tinycloud-node-server/tests/postgres_connection_recovery.rs +++ b/tinycloud-node-server/tests/postgres_connection_recovery.rs @@ -804,38 +804,32 @@ async fn postgres_requests_survive_one_stale_pooled_connection() { fixture.drop_schema().await; } -/// Candidate registration is an idempotent write before upload. A stale idle -/// connection must be retried here, before any blob can land in storage. +/// Ordinary KV writes begin with an idempotent preparatory transaction. +/// A stale idle connection must not produce an orphan or prevent the write. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn postgres_preupload_candidate_registration_retries_stale_connection() { - let Some(url) = - postgres_url("postgres_preupload_candidate_registration_retries_stale_connection") +async fn postgres_ordinary_put_recovers_stale_connection_without_orphan() { + let Some(url) = postgres_url("postgres_ordinary_put_recovers_stale_connection_without_orphan") else { return; }; - let fixture = Fixture::new("candidate", &url, None).await; + let mut fixture = Fixture::new("candidate", &url, None).await; fixture.fill_pool().await; let victim = fixture.order_idle_queue(0).await; fixture.terminate(&BTreeSet::from([victim])).await; let before = fixture.stats(); - let blob = ( - fixture.owner.space.clone(), - tinycloud_core::hash::hash(b"preupload-stale-connection"), - ); - tinycloud_core::kv_orphans::register_candidates_idempotent(&fixture.conn, &[blob]) + fixture + .put("after/preflight/stale") .await - .expect("candidate is registered after reconnect"); + .expect("ordinary write survives stale pooled connection"); let recovered = delta(before, fixture.stats()); assert_eq!( - ( - recovered.losses, - recovered.retries, - recovered.idempotent_retries - ), - (1, 1, 1), + (recovered.losses, recovered.retries), + (1, 1), "{recovered:?}" ); - assert_eq!(fixture.row_count("kv_orphan_candidate").await, 1); + assert_eq!(fixture.row_count("kv_orphan_candidate").await, 0); + assert_eq!(fixture.row_count("kv_orphan_attempt").await, 0); + fixture.get("after/preflight/stale").await.unwrap(); fixture.drop_schema().await; } diff --git a/tinycloud-node-server/tests/postgres_kv_write_throughput.rs b/tinycloud-node-server/tests/postgres_kv_write_throughput.rs index 24ff22da..4d5f9b6d 100644 --- a/tinycloud-node-server/tests/postgres_kv_write_throughput.rs +++ b/tinycloud-node-server/tests/postgres_kv_write_throughput.rs @@ -568,11 +568,16 @@ async fn postgres_s3_orphan_cleanup_reclaims_space_usage() { let mut stage = HashBuffer::new(Vec::new()); stage.write_all(b"tc735-orphan-cleanup").await.unwrap(); let hash = stage.hash(); - tinycloud_core::kv_orphans::register_candidates(&conn, &[(owner.space.clone(), hash)]) - .await - .unwrap(); + let first = tinycloud_core::kv_orphans::register_candidates( + &conn, + &[(owner.space.clone(), hash)], + "crashed", + ) + .await + .unwrap(); + assert_eq!(first[0].2, 0, "the first upload uses the legacy bare key"); assert_eq!( - ImmutableWriteStore::::persist(&store, &owner.space, stage) + ImmutableWriteStore::::persist(&store, &owner.space, stage, first[0].2) .await .unwrap(), hash @@ -584,7 +589,10 @@ async fn postgres_s3_orphan_cleanup_reclaims_space_usage() { .unwrap(), 1 ); - assert!(!store.contains(&owner.space, &hash).await.unwrap()); + assert!(!store + .contains(&owner.space, &hash, first[0].2) + .await + .unwrap()); assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); // Simulate a known failure whose object removal succeeded but whose // candidate-row deletion did not commit. A later GC pass must not debit @@ -593,19 +601,30 @@ async fn postgres_s3_orphan_cleanup_reclaims_space_usage() { failed.write_all(b"tc735-known-failure").await.unwrap(); let failed_hash = failed.hash(); let candidate = [(owner.space.clone(), failed_hash)]; - tinycloud_core::kv_orphans::register_candidates(&conn, &candidate) - .await - .unwrap(); - ImmutableWriteStore::::persist(&store, &owner.space, failed) - .await - .unwrap(); - tinycloud_core::kv_orphans::refund_failed(&conn, &store, &candidate) + let failed_attempt = + tinycloud_core::kv_orphans::register_candidates(&conn, &candidate, "known-failure") + .await + .unwrap(); + ImmutableWriteStore::::persist( + &store, + &owner.space, + failed, + failed_attempt[0].2, + ) + .await + .unwrap(); + tinycloud_core::kv_orphans::refund_failed(&conn, &store, &failed_attempt, "known-failure") .await .unwrap(); assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); - tinycloud_core::kv_orphans::register_candidates(&conn, &candidate) - .await - .unwrap(); + let retry = + tinycloud_core::kv_orphans::register_candidates(&conn, &candidate, "retry-after-delete") + .await + .unwrap(); + assert_eq!( + retry[0].2, 1, + "confirmed deletion advances storage generation" + ); assert_eq!( tinycloud_core::kv_orphans::collect_candidates(&conn, &store, 1, time::Duration::ZERO,) .await @@ -613,7 +632,10 @@ async fn postgres_s3_orphan_cleanup_reclaims_space_usage() { 0, "the stale candidate has no physical object left to remove" ); - assert!(!store.contains(&owner.space, &failed_hash).await.unwrap()); + assert!(!store + .contains(&owner.space, &failed_hash, retry[0].2) + .await + .unwrap()); assert_eq!(store.total_size(&owner.space).await.unwrap(), Some(0)); conn.close().await.unwrap(); admin diff --git a/tinycloud-node-server/tests/postgres_round_trip_budget.rs b/tinycloud-node-server/tests/postgres_round_trip_budget.rs index 6c469472..dad92697 100644 --- a/tinycloud-node-server/tests/postgres_round_trip_budget.rs +++ b/tinycloud-node-server/tests/postgres_round_trip_budget.rs @@ -124,12 +124,15 @@ use tokio::{ /// net new statements are seven necessary steps minus three queries saved by /// #302 in the commit authorization graph: BEGIN/COMMIT pin pre-upload work /// for retry-safe atomicity (two), the pre-upload authorization graph and -/// hosted-space COUNT reject before S3 (two), the candidate reservation -/// survives a crash before upload (one), a second hash-fence reservation -/// under the commit transaction excludes GC after upload (one), and the -/// candidate DELETE clears the committed reference atomically (one). -/// The extra preflight transaction also releases one pooled connection, -/// whose ping flushes cancelled/unfinished statements and cannot be removed. +/// hosted-space COUNT reject before S3 (two), the PostgreSQL candidate-and- +/// attempt registration CTE durably protects the blob before upload (one), +/// the joined active-fence/own-attempt lock excludes GC after upload (one), +/// and the successful-commit CTE retires the attempt and only an already +/// referenced last fence (one). A failed last attempt instead persists a +/// deleting fence until physical absence is confirmed. The extra preparatory +/// transaction releases one pooled connection; its mandatory ping flushes +/// cancelled/unfinished statements. SQLite/MySQL use the equivalent guarded +/// multi-statement paths rather than the PostgreSQL-only CTEs. const BUDGETS: [(&str, u64); 3] = [("sign-in", 69), ("kv put", 42), ("kv get", 16)]; /// Full sign-in plus put plus get rounds run before the measured one. @@ -1188,11 +1191,13 @@ async fn postgres_write_routes_tell_retryable_failures_from_unknown_outcomes() { let (status, body, _) = kv_put(&node, &session, "tc1054/warm").await?; ensure!(status == Status::Ok, "warm-up put: {status} {body}"); - // KV put: no connection survives BEGIN, so nothing was written. + // KV put's preparatory BEGIN retries once in the recovering pool and + // once around the idempotent preparation (two attempts each). Exhaust + // both layers before asserting the public retryable response. node.settle().await?; - cut.arm(&[b"BEGIN"], 2); + cut.arm(&[b"BEGIN"], 4); let (status, body, retry_after) = kv_put(&node, &session, "tc1054/retryable").await?; - ensure!(!cut.armed(), "both BEGIN attempts were cut"); + ensure!(!cut.armed(), "all preparatory BEGIN attempts were cut"); ensure!( status == Status::ServiceUnavailable && body.starts_with("unavailable_retryable:"), "kv put before any write: {status} {body}" From 2a37bfc256bbdfebf0680c38cda24ca7788f297d Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 01:21:44 +0000 Subject: [PATCH 06/17] fix: serialize blob generations and publish filesystem bytes atomically --- CHANGELOG.md | 1 + docs/kv-sync.md | 9 + tinycloud-core/src/db.rs | 25 ++ tinycloud-core/src/db_pool.rs | 13 +- .../migrations/m20260724_010000_current_kv.rs | 68 +++- .../src/storage/file_system.rs | 317 ++++++++++++++++-- tinycloud-node-server/src/storage/mod.rs | 32 ++ tinycloud-node-server/src/storage/s3.rs | 181 +++++++++- 8 files changed, 604 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f3e158e..62d0c15c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. + Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key; crash-leftover stages count at restart and are reclaimed with their candidate. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/docs/kv-sync.md b/docs/kv-sync.md index a4f5977d..a8c76676 100644 --- a/docs/kv-sync.md +++ b/docs/kv-sync.md @@ -147,6 +147,15 @@ authorization and conditional-write preconditions. The guards stay through commit, so `seq` and sync cursors remain commit-ordered even when same-space uploads finish out of order. An unauthorized write uploads nothing. +For a single node process, each `(space, hash, generation)` serializes its +physical existence check, upload or deletion, and quota settlement without +holding a database connection. Filesystem memory staging writes into a +same-directory temporary file and atomically renames only complete bytes to +the hash key. A process crash can leave an incomplete stage; startup counts +its physical bytes, and the hash candidate's GC pass removes and refunds it. +SQLite preparatory writes reserve the database writer before authorization +reads, so a separate collector connection cannot invalidate the read snapshot. + Protected legacy meeting paths are the TC-1260 exception: the durable freeze row and its pooled transaction remain locked across upload until commit or abort. A freeze acknowledgement drains earlier uploads; a later protected diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index 4dac0ebc..f224b884 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -4980,6 +4980,31 @@ mod test { .unwrap(); } + #[tokio::test] + async fn sqlite_preupload_registration_reserves_writer_before_authorization_reads() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("candidate.db"); + let conn = sqlite_write_pool(&path, std::time::Duration::from_secs(5), 1).await; + let collector = sqlite_write_pool(&path, std::time::Duration::from_millis(100), 1).await; + let recovering = RecoveringConnection::new(conn); + + let prep = recovering.begin_preparatory().await.unwrap(); + epoch::Entity::find().count(&prep).await.unwrap(); + let error = test_actor("did:key:collector") + .insert(&collector) + .await + .expect_err("collector cannot invalidate the preparatory read snapshot"); + assert_eq!(sqlite_error_code(&error).as_deref(), Some("5"), "{error}"); + prep.execute_unprepared(SQLITE_RESERVE_WRITE_LOCK) + .await + .expect("registration after authorization remains writable"); + prep.commit().await.unwrap(); + test_actor("did:key:collector") + .insert(&collector) + .await + .unwrap(); + } + #[tokio::test] async fn postgres_write_transaction_is_a_plain_begin() { let Some(database_url) = diff --git a/tinycloud-core/src/db_pool.rs b/tinycloud-core/src/db_pool.rs index ed654106..5e6f7c01 100644 --- a/tinycloud-core/src/db_pool.rs +++ b/tinycloud-core/src/db_pool.rs @@ -257,12 +257,17 @@ impl RecoveringConnection { } /// Pre-upload authorization and orphan registration share one connection. - /// This transaction can only change retryable candidate metadata; the - /// caller commits it before persisting a blob, so its uncertain outcome - /// cannot mean a user-visible KV write committed. + /// Reserve SQLite's database writer before authorization reads, just like + /// the other write paths: a separate collector connection must not commit + /// after our snapshot and turn registration into SQLITE_BUSY_SNAPSHOT. + /// This transaction changes only retryable candidate metadata and commits + /// before blob persistence, so its uncertain outcome cannot mean a + /// user-visible KV write committed. pub(crate) async fn begin_preparatory(&self) -> Result { let transaction = self - .run("begin_preparatory", true, || self.conn.begin()) + .run("begin_preparatory", true, || { + crate::db::begin_write_transaction(&self.conn, None) + }) .await?; self.scope_local_timeout(transaction).await } diff --git a/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs b/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs index 1104e3cb..54f1c614 100644 --- a/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs +++ b/tinycloud-core/src/migrations/m20260724_010000_current_kv.rs @@ -119,7 +119,7 @@ fn latest_writes() -> SelectStatement { (write.clone(), kv_write::Column::Metadata), ]) .expr_as(Expr::exists(exact_tombstone), current_kv::Column::Deleted) - .expr_as(Expr::val(0), current_kv::Column::Generation) + .expr_as(Expr::val(0_i64), current_kv::Column::Generation) .from_as(kv_write::Entity, write) .cond_where(Condition::all().not().add(Expr::exists(newer_write))); query.to_owned() @@ -177,7 +177,10 @@ mod tests { use crate::models::{actor, epoch, invocation, space}; use crate::relationships::event_order; use crate::types::{Metadata, SpaceIdWrap}; - use sea_orm::{ActiveModelTrait, ActiveValue::Set, ConnectOptions, Database, PaginatorTrait}; + use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ConnectOptions, ConnectionTrait, Database, DbBackend, + PaginatorTrait, Statement, + }; use sea_orm_migration::MigratorTrait; use std::collections::BTreeMap; use time::OffsetDateTime; @@ -343,4 +346,65 @@ mod tests { "backfill must preserve immutable delete history" ); } + #[tokio::test] + async fn postgres_populated_pre_current_kv_backfill_decodes_generation() { + let Some(url) = crate::test_support::postgres_test_url( + "postgres_populated_pre_current_kv_backfill_decodes_generation", + ) else { + return; + }; + let admin = Database::connect(url.clone()).await.unwrap(); + let schema = format!( + "tc735_old_kv_{}_{}", + std::process::id(), + OffsetDateTime::now_utc().unix_timestamp_nanos() + ); + admin + .execute(Statement::from_string( + DbBackend::Postgres, + format!("CREATE SCHEMA {schema}"), + )) + .await + .unwrap(); + let mut options = ConnectOptions::new(url); + options.set_schema_search_path(schema.clone()); + let db = Database::connect(options).await.unwrap(); + Migrator::up(&db, Some(13)).await.unwrap(); + let did: DIDBuf = DID_METHODS + .generate(&JWK::generate_ed25519().unwrap(), "key") + .unwrap(); + let space_id = SpaceId::new(did, "migration".parse().unwrap()); + actor::ActiveModel { + id: Set("did:key:migration".to_string()), + } + .insert(&db) + .await + .unwrap(); + space::ActiveModel { + id: Set(SpaceIdWrap(space_id.clone())), + } + .insert(&db) + .await + .unwrap(); + let write = insert_write(&db, &space_id, "older-key", "older-value", 1).await; + let backfill = Migration.up(&SchemaManager::new(&db)).await; + let current = if backfill.is_ok() { + current_kv::Entity::find().one(&db).await.unwrap() + } else { + None + }; + db.close().await.unwrap(); + admin + .execute(Statement::from_string( + DbBackend::Postgres, + format!("DROP SCHEMA {schema} CASCADE"), + )) + .await + .unwrap(); + backfill.unwrap(); + let current = current.unwrap(); + assert_eq!(current.invocation, write.invocation); + assert_eq!(current.value, write.value); + assert_eq!(current.generation, 0); + } } diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index 7ee28813..5566471d 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -1,4 +1,6 @@ -use super::{size::SpaceSizes, BlobGenerationKey}; +#[cfg(test)] +use super::TestPause; +use super::{size::SpaceSizes, BlobGenerationKey, BlobOperationLocks}; use core::pin::Pin; use futures::{ future::{Either as AsyncEither, TryFutureExt}, @@ -19,6 +21,8 @@ use std::{ use tempfile::{NamedTempFile, PathPersistError}; use tinycloud_auth::{resource::SpaceId, ssi::dids::DIDBuf}; use tinycloud_core::{hash::Hash, storage::*}; +#[cfg(test)] +use tokio::sync::Notify; use tokio::{ fs::{create_dir_all, metadata, remove_file, File}, io::{AsyncReadExt, AsyncSeekExt}, @@ -32,9 +36,15 @@ use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; pub struct FileSystemStore { path: PathBuf, sizes: SpaceSizes, - // A partial filesystem write can fail after creating a file. Reserve - // before writing and refund the full reservation when deleting it. + // Reserve the full size before writing a temporary file. A cancellation + // or failed publish leaves a conservative charge until cleanup confirms + // that neither the stage nor the final object remains. pending: Arc>>, + operations: BlobOperationLocks, + #[cfg(test)] + before_memory_write: TestPause, + #[cfg(test)] + after_memory_contains: TestPause, } impl FileSystemStore { @@ -45,6 +55,11 @@ impl FileSystemStore { path, sizes, pending: Arc::new(Mutex::new(HashMap::new())), + operations: BlobOperationLocks::default(), + #[cfg(test)] + before_memory_write: Arc::new(Mutex::new(None)), + #[cfg(test)] + after_memory_contains: Arc::new(Mutex::new(None)), }) } @@ -81,15 +96,55 @@ impl FileSystemStore { .remove(&(space.clone(), hash, generation)); } - async fn refund_missing_pending(&self, space: &SpaceId, hash: Hash, generation: i64) { - if let Some(size) = self - .pending - .lock() - .await - .remove(&(space.clone(), hash, generation)) - { - self.decrement_size(space, size).await; + fn stage_prefix(path: &Path) -> String { + format!( + ".tc-stage-{}-", + path.file_name() + .expect("blob path has a name") + .to_string_lossy() + ) + } + + /// A process crash can leave a named temporary file in a space directory. + /// Boot quota reconstruction counts its actual bytes; candidate GC removes + /// files for that hash and refunds them along with any published object. + async fn remove_stages( + &self, + space: &SpaceId, + path: &Path, + refund_each: bool, + ) -> Result { + let Some(parent) = path.parent() else { + return Ok(0); + }; + let mut entries = match tokio::fs::read_dir(parent).await { + Ok(entries) => entries, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(error), + }; + let prefix = Self::stage_prefix(path); + let mut removed = 0u64; + while let Some(entry) = entries.next_entry().await? { + if !entry.file_name().to_string_lossy().starts_with(&prefix) + || !entry.file_type().await?.is_file() + { + continue; + } + let size = entry.metadata().await?.len(); + match remove_file(entry.path()).await { + Ok(()) => { + removed = removed + .checked_add(size) + .ok_or_else(|| IoError::other("staged blob byte count overflow"))?; + if refund_each { + self.decrement_size(space, size).await; + } + } + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(error), + } } + Ok(removed) } async fn persist_temp( @@ -99,6 +154,10 @@ impl FileSystemStore { generation: i64, file: TempFileStage, ) -> Result<(), FileSystemStoreError> { + let _operation = self + .operations + .lock((space.clone(), hash, generation)) + .await; if self.contains(space, &hash, generation).await? { self.settle(space, hash, generation).await; return Ok(()); @@ -118,16 +177,42 @@ impl FileSystemStore { generation: i64, bytes: Vec, ) -> Result<(), FileSystemStoreError> { + let _operation = self + .operations + .lock((space.clone(), hash, generation)) + .await; if self.contains(space, &hash, generation).await? { self.settle(space, hash, generation).await; return Ok(()); } + #[cfg(test)] + let pause = self.after_memory_contains.lock().await.take(); + #[cfg(test)] + if let Some((entered, release)) = pause { + entered.notify_one(); + release.notified().await; + } self.reserve(space, hash, generation, bytes.len() as u64) .await; - let file = File::create(self.get_path(space, &hash, generation)).await?; - let mut writer = futures::io::BufWriter::new(file.compat()); + let target = self.get_path(space, &hash, generation); + // Stage on the target filesystem so the final rename is atomic and + // never depends on a cross-device copy. + let staged = tempfile::Builder::new() + .prefix(&Self::stage_prefix(&target)) + .tempfile_in(target.parent().expect("blob path has a parent"))?; + let (file, path) = staged.into_parts(); + #[cfg(test)] + let pause = self.before_memory_write.lock().await.take(); + #[cfg(test)] + if let Some((entered, release)) = pause { + entered.notify_one(); + release.notified().await; + } + let mut writer = File::from_std(file).compat(); writer.write_all(&bytes).await?; writer.flush().await?; + drop(writer); + path.persist(target)?; self.settle(space, hash, generation).await; Ok(()) } @@ -447,32 +532,38 @@ impl ImmutableDeleteStore for FileSystemStore { id: &Hash, generation: i64, ) -> Result, Self::Error> { + let key = (space.clone(), *id, generation); + let _operation = self.operations.lock(key.clone()).await; let path = self.get_path(space, id, generation); - let size = match metadata(&path).await { - Ok(m) => m.len(), - Err(e) if e.kind() == ErrorKind::NotFound => { - self.refund_missing_pending(space, *id, generation).await; - return Ok(None); - } - Err(e) => return Err(e.into()), + let pending = self.pending.lock().await.contains_key(&key); + // A crash can leave a named, incomplete memory stage. Boot counted + // its actual bytes; a live reservation instead counted its full size. + let staged = self.remove_stages(space, &path, !pending).await?; + let final_size = match metadata(&path).await { + Ok(meta) => Some(meta.len()), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), }; - match remove_file(path).await { - Ok(()) => { - let charged = self - .pending - .lock() - .await - .remove(&(space.clone(), *id, generation)) - .unwrap_or(size); - self.decrement_size(space, charged).await; - Ok(Some(())) + let removed_final = if let Some(size) = final_size { + match remove_file(&path).await { + Ok(()) => Some(size), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), } - Err(e) if e.kind() == ErrorKind::NotFound => { - self.refund_missing_pending(space, *id, generation).await; - Ok(None) - } - Err(e) => Err(e.into()), + } else { + None + }; + if pending { + let reserved = self.pending.lock().await.remove(&key).unwrap_or(0); + self.decrement_size( + space, + reserved.max(staged.saturating_add(removed_final.unwrap_or(0))), + ) + .await; + } else if let Some(size) = removed_final { + self.decrement_size(space, size).await; } + Ok((removed_final.is_some() || staged != 0).then_some(())) } } @@ -489,6 +580,162 @@ mod test { use super::*; use futures::io::{AsyncReadExt, AsyncWriteExt}; + #[tokio::test] + async fn memory_publish_is_atomic_for_concurrent_same_hash_paths() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:atomic".parse().unwrap(); + store.create(&space).await.unwrap(); + let data = b"same-content two distinct KV paths".to_vec(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.before_memory_write.lock().await = Some((entered.clone(), release)); + let mut first_stage = HashBuffer::new(Vec::new()); + first_stage.write_all(&data).await.unwrap(); + let hash = first_stage.hash(); + let first_store = store.clone(); + let first_space = space.clone(); + let first = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &first_store, + &first_space, + first_stage, + 0, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + let prematurely_visible = store.contains(&space, &hash, 0).await.unwrap(); + let mut second_stage = HashBuffer::new(Vec::new()); + second_stage.write_all(&data).await.unwrap(); + let second_store = store.clone(); + let second_space = space.clone(); + let mut second = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &second_store, + &second_space, + second_stage, + 0, + ) + .await + }); + let early_second = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut second).await; + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let second_hash = match early_second { + Ok(result) => result.unwrap().unwrap(), + Err(_) => tokio::time::timeout(std::time::Duration::from_secs(2), second) + .await + .unwrap() + .unwrap() + .unwrap(), + }; + assert_eq!(second_hash, hash); + let stored = store.read_to_vec(&space, &hash, 0).await.unwrap().unwrap(); + assert!( + !prematurely_visible, + "a complete candidate must be published before same-hash dedup" + ); + assert_eq!(stored, data, "a committed alias must read all staged bytes"); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(data.len() as u64) + ); + } + + #[tokio::test] + async fn same_generation_concurrent_memory_put_charges_once() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:one-charge".parse().unwrap(); + store.create(&space).await.unwrap(); + let data = b"same content for two KV records".to_vec(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.after_memory_contains.lock().await = Some((entered.clone(), release.clone())); + let mut first_stage = HashBuffer::new(Vec::new()); + first_stage.write_all(&data).await.unwrap(); + let hash = first_stage.hash(); + let first_store = store.clone(); + let first_space = space.clone(); + let first = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &first_store, + &first_space, + first_stage, + 0, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + let mut second_stage = HashBuffer::new(Vec::new()); + second_stage.write_all(&data).await.unwrap(); + let second_store = store.clone(); + let second_space = space.clone(); + let mut second = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &second_store, + &second_space, + second_stage, + 0, + ) + .await + }); + let early_second = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut second).await; + release.notify_one(); + assert_eq!(first.await.unwrap().unwrap(), hash); + let second_hash = match early_second { + Ok(result) => result.unwrap().unwrap(), + Err(_) => tokio::time::timeout(std::time::Duration::from_secs(2), second) + .await + .unwrap() + .unwrap() + .unwrap(), + }; + assert_eq!(second_hash, hash); + assert_eq!( + store.read_to_vec(&space, &hash, 0).await.unwrap(), + Some(data.clone()) + ); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(data.len() as u64) + ); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), Some(())); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn restart_counts_then_collects_crash_leftover_memory_stage() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:stage-restart".parse().unwrap(); + store.create(&space).await.unwrap(); + let hash = tinycloud_core::hash::hash(b"unfinished memory stage"); + let path = store.get_path(&space, &hash, 0); + let prefix = format!(".tc-stage-{}-", path.file_name().unwrap().to_str().unwrap()); + let mut stage = tempfile::Builder::new() + .prefix(&prefix) + .tempfile_in(path.parent().unwrap()) + .unwrap(); + std::io::Write::write_all(&mut stage, b"partial").unwrap(); + let stage_path = stage.path().to_path_buf(); + stage.into_temp_path().keep().unwrap(); + drop(store); + + let reopened = FileSystemConfig::new(dir.path()).open().await.unwrap(); + assert_eq!(reopened.total_size(&space).await.unwrap(), Some(7)); + reopened.remove(&space, &hash, 0).await.unwrap(); + assert_eq!(reopened.total_size(&space).await.unwrap(), Some(0)); + assert!(!stage_path.exists()); + } + #[tokio::test] async fn failed_file_creation_refunds_reserved_quota() { let dir = tempfile::tempdir().unwrap(); diff --git a/tinycloud-node-server/src/storage/mod.rs b/tinycloud-node-server/src/storage/mod.rs index b665d262..802dc7e4 100644 --- a/tinycloud-node-server/src/storage/mod.rs +++ b/tinycloud-node-server/src/storage/mod.rs @@ -5,6 +5,38 @@ pub mod size; use tinycloud_auth::resource::SpaceId; use tinycloud_core::hash::Hash; +use std::{ + collections::HashMap, + sync::{Arc, Weak}, +}; +use tokio::sync::{Mutex, OwnedMutexGuard}; + /// One physical blob lifetime; a later generation must never share its /// quota reservation or deletion target with an earlier generation. type BlobGenerationKey = (SpaceId, Hash, i64); +#[cfg(test)] +type TestPause = Arc, Arc)>>>; + +/// Serialize physical I/O for one generation, not the entire store. The weak +/// registry drops inactive keys, so a long-running node does not retain one +/// lock per content hash it has ever written or collected. No DB connection +/// is held while this guard waits or while an object-store request runs. +#[derive(Debug, Clone, Default)] +struct BlobOperationLocks(Arc>>>>); + +impl BlobOperationLocks { + async fn lock(&self, key: BlobGenerationKey) -> OwnedMutexGuard<()> { + let lock = { + let mut registry = self.0.lock().await; + registry.retain(|_, weak| weak.strong_count() > 0); + if let Some(lock) = registry.get(&key).and_then(Weak::upgrade) { + lock + } else { + let lock = Arc::new(Mutex::new(())); + registry.insert(key, Arc::downgrade(&lock)); + lock + } + }; + lock.lock_owned().await + } +} diff --git a/tinycloud-node-server/src/storage/s3.rs b/tinycloud-node-server/src/storage/s3.rs index f20ecad2..0fe499e5 100644 --- a/tinycloud-node-server/src/storage/s3.rs +++ b/tinycloud-node-server/src/storage/s3.rs @@ -25,8 +25,12 @@ use std::{ use tinycloud_auth::resource::SpaceId; use tinycloud_core::{hash::Hash, storage::*}; -use super::{file_system, size::SpaceSizes, BlobGenerationKey}; +#[cfg(test)] +use super::TestPause; +use super::{file_system, size::SpaceSizes, BlobGenerationKey, BlobOperationLocks}; use tokio::sync::Mutex; +#[cfg(test)] +use tokio::sync::Notify; async fn aws_config() -> SdkConfig { aws_config::from_env().load().await @@ -45,6 +49,11 @@ pub struct S3BlockStore { // size until absence is confirmed, so retry refunds exactly once. On // restart, the initial bucket listing reconstructs only physical bytes. deleting: Arc>>, + operations: BlobOperationLocks, + #[cfg(test)] + after_delete_head: TestPause, + #[cfg(test)] + after_put_head: TestPause, } #[serde_as] @@ -155,6 +164,11 @@ impl S3BlockStore { sizes, pending: Arc::new(Mutex::new(HashMap::new())), deleting: Arc::new(Mutex::new(counted_deletes)), + operations: BlobOperationLocks::default(), + #[cfg(test)] + after_delete_head: Arc::new(Mutex::new(None)), + #[cfg(test)] + after_put_head: Arc::new(Mutex::new(None)), }) } @@ -194,11 +208,21 @@ impl S3BlockStore { Fut: Future>, { let key = (space.clone(), hash, generation); + // Dedup, reservation, PUT and settlement are one physical generation + // operation. A second KV path cannot charge the same bytes again. + let _operation = self.operations.lock(key.clone()).await; if self.contains(space, &hash, generation).await? { // A retry of an ambiguous PUT can find its original object. self.pending.lock().await.remove(&key); return Ok(()); } + #[cfg(test)] + let pause = self.after_put_head.lock().await.take(); + #[cfg(test)] + if let Some((entered, release)) = pause { + entered.notify_one(); + release.notified().await; + } let (size, body, guard) = body().await?; let mut pending = self.pending.lock().await; if let Entry::Vacant(slot) = pending.entry(key.clone()) { @@ -539,6 +563,9 @@ impl ImmutableDeleteStore for S3BlockStore { id: &Hash, generation: i64, ) -> Result, Self::Error> { + // A stale positive HEAD must not recreate a charge after another + // deleter has already confirmed absence and refunded it. + let _operation = self.operations.lock((space.clone(), *id, generation)).await; let size: u64 = match self .client .head_object() @@ -561,6 +588,13 @@ impl ImmutableDeleteStore for S3BlockStore { } Err(error) => return Err(S3Error::from(error).into()), }; + #[cfg(test)] + let pause = self.after_delete_head.lock().await.take(); + #[cfg(test)] + if let Some((entered, release)) = pause { + entered.notify_one(); + release.notified().await; + } self.deleting .lock() .await @@ -693,6 +727,151 @@ mod tests { } } + #[tokio::test] + async fn minio_same_generation_concurrent_put_charges_once() { + use futures::io::AsyncWriteExt; + let Ok(endpoint) = std::env::var("TC735_TEST_MINIO_ENDPOINT") else { + return; + }; + let bucket = std::env::var("TC735_TEST_MINIO_BUCKET") + .expect("TC735_TEST_MINIO_BUCKET is required with TC735_TEST_MINIO_ENDPOINT"); + let store = S3BlockConfig { + bucket, + endpoint: Some(endpoint.parse().unwrap()), + } + .open() + .await + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let space: SpaceId = format!("tinycloud:key:test:tc735-same-charge-{nonce}") + .parse() + .unwrap(); + store.create(&space).await.unwrap(); + let data = b"same bytes under two KV paths"; + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.after_put_head.lock().await = Some((entered.clone(), release.clone())); + let mut first_stage = memory::MemoryStaging.stage(&space).await.unwrap(); + first_stage.write_all(data).await.unwrap(); + let hash = first_stage.hash(); + let first_store = store.clone(); + let first_space = space.clone(); + let first = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &first_store, + &first_space, + first_stage, + 0, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(3), entered.notified()) + .await + .unwrap(); + let mut second_stage = memory::MemoryStaging.stage(&space).await.unwrap(); + second_stage.write_all(data).await.unwrap(); + let second_store = store.clone(); + let second_space = space.clone(); + let mut second = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &second_store, + &second_space, + second_stage, + 0, + ) + .await + }); + let early_second = + tokio::time::timeout(std::time::Duration::from_secs(2), &mut second).await; + release.notify_one(); + assert_eq!(first.await.unwrap().unwrap(), hash); + let second_hash = match early_second { + Ok(result) => result.unwrap().unwrap(), + Err(_) => second.await.unwrap().unwrap(), + }; + assert_eq!(second_hash, hash); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(data.len() as u64) + ); + assert_eq!( + store.read_to_vec(&space, &hash, 0).await.unwrap(), + Some(data.to_vec()) + ); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), Some(())); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn minio_stale_head_cannot_recreate_refunded_deletion_charge() { + use futures::io::AsyncWriteExt; + let Ok(endpoint) = std::env::var("TC735_TEST_MINIO_ENDPOINT") else { + return; + }; + let bucket = std::env::var("TC735_TEST_MINIO_BUCKET") + .expect("TC735_TEST_MINIO_BUCKET is required with TC735_TEST_MINIO_ENDPOINT"); + let store = S3BlockConfig { + bucket, + endpoint: Some(endpoint.parse().unwrap()), + } + .open() + .await + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let space: SpaceId = format!("tinycloud:key:test:tc735-stale-head-{nonce}") + .parse() + .unwrap(); + store.create(&space).await.unwrap(); + let data = b"same generation charge"; + let mut stage = memory::MemoryStaging.stage(&space).await.unwrap(); + stage.write_all(data).await.unwrap(); + let hash = ImmutableWriteStore::::persist(&store, &space, stage, 0) + .await + .unwrap(); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(data.len() as u64) + ); + + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.after_delete_head.lock().await = Some((entered.clone(), release.clone())); + let first_store = store.clone(); + let first_space = space.clone(); + let first = tokio::spawn(async move { first_store.remove(&first_space, &hash, 0).await }); + tokio::time::timeout(std::time::Duration::from_secs(3), entered.notified()) + .await + .unwrap(); + let second_store = store.clone(); + let second_space = space.clone(); + let mut second = + tokio::spawn(async move { second_store.remove(&second_space, &hash, 0).await }); + // Without per-generation serialization, second DELETE completes + // while the first retains a stale positive HEAD. With the fix, it + // waits and then observes the confirmed absence. + let early_second = + tokio::time::timeout(std::time::Duration::from_secs(2), &mut second).await; + release.notify_one(); + let first_result = first.await.unwrap().unwrap(); + let second_result = match early_second { + Ok(result) => result.unwrap().unwrap(), + Err(_) => second.await.unwrap().unwrap(), + }; + assert!(first_result.is_some() || second_result.is_some()); + assert!(!store.contains(&space, &hash, 0).await.unwrap()); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(0), + "the same generation can refund physical bytes only once" + ); + } + #[tokio::test] async fn minio_lost_delete_response_refunds_once_and_restart_reconstructs() { use futures::io::AsyncWriteExt; From 51a9125870ee11c80674747710a6dc79b90571df Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 02:11:17 +0000 Subject: [PATCH 07/17] fix: reclaim referenced crash-leftover filesystem stages --- CHANGELOG.md | 2 +- tinycloud-core/src/db.rs | 8 + tinycloud-core/src/kv_orphans.rs | 30 +- tinycloud-core/src/kv_sync.rs | 8 + tinycloud-core/src/storage/either.rs | 17 ++ tinycloud-core/src/storage/memory.rs | 9 + tinycloud-core/src/storage/mod.rs | 18 ++ .../src/storage/file_system.rs | 257 ++++++++++++++++++ tinycloud-node-server/src/storage/s3.rs | 9 + 9 files changed, 356 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d0c15c..f9cda064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. - Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key; crash-leftover stages count at restart and are reclaimed with their candidate. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. + Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index f224b884..f633345e 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -5663,6 +5663,14 @@ mod test { } self.inner.remove(space, hash, generation).await } + async fn cleanup_stages( + &self, + space: &SpaceId, + hash: &Hash, + generation: i64, + ) -> Result<(), Self::Error> { + self.inner.cleanup_stages(space, hash, generation).await + } } #[tokio::test] async fn legacy_freeze_ack_waits_for_actual_in_flight_kv_storage_and_commit() { diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index f7706a07..c831cdd3 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -422,6 +422,8 @@ pub enum CollectError { Database(#[from] DbErr), #[error("orphan blob metadata read failed: {0}")] Read(String), + #[error("orphan stage cleanup timed out")] + StageTimeout, #[error("orphan blob deletion failed: {0}")] Store(E), } @@ -571,10 +573,28 @@ where return Ok(false); } if has_attempts(&tx, &key, value, generation).await? { - tx.commit().await?; + if skip_locked { + // GC must retain the expired claim while another upload is + // in flight. Its crashed stage is still owed cleanup even if the + // other writer commits and would otherwise retire the fence. + tx.rollback().await?; + } else { + tx.commit().await?; + } return Ok(false); } if has_references(&tx, &key, value, generation).await? { + // This is a rare local-filesystem cleanup, not a blob upload. Keep + // the fence locked until only incomplete stages are gone, then retire + // the candidate. A timeout or cancellation rolls this transaction + // back, preserving its attempt for the next GC pass. + tokio::time::timeout( + std::time::Duration::from_secs(5), + store.cleanup_stages(space, &value, generation), + ) + .await + .map_err(|_| CollectError::StageTimeout)? + .map_err(CollectError::Store)?; candidate::Entity::delete_many() .filter(candidate::Column::Space.eq(key)) .filter(candidate::Column::Value.eq(value)) @@ -952,6 +972,14 @@ mod tests { } Ok(removed) } + async fn cleanup_stages( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result<(), Self::Error> { + self.inner.cleanup_stages(space, id, generation).await + } } async fn database() -> DatabaseConnection { diff --git a/tinycloud-core/src/kv_sync.rs b/tinycloud-core/src/kv_sync.rs index cc09a62f..4001e4fb 100644 --- a/tinycloud-core/src/kv_sync.rs +++ b/tinycloud-core/src/kv_sync.rs @@ -1933,6 +1933,14 @@ pub(crate) mod tests { ) -> Result, Self::Error> { self.0.remove(space, id, generation).await } + async fn cleanup_stages( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result<(), Self::Error> { + self.0.cleanup_stages(space, id, generation).await + } } #[tokio::test] diff --git a/tinycloud-core/src/storage/either.rs b/tinycloud-core/src/storage/either.rs index 0e2befa5..0794d721 100644 --- a/tinycloud-core/src/storage/either.rs +++ b/tinycloud-core/src/storage/either.rs @@ -236,6 +236,23 @@ where .map_err(Self::Error::B), } } + async fn cleanup_stages( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result<(), Self::Error> { + match self { + Self::A(store) => store + .cleanup_stages(space, id, generation) + .await + .map_err(Self::Error::A), + Self::B(store) => store + .cleanup_stages(space, id, generation) + .await + .map_err(Self::Error::B), + } + } } #[async_trait] diff --git a/tinycloud-core/src/storage/memory.rs b/tinycloud-core/src/storage/memory.rs index e43ed9e5..4018d299 100644 --- a/tinycloud-core/src/storage/memory.rs +++ b/tinycloud-core/src/storage/memory.rs @@ -223,6 +223,15 @@ impl ImmutableDeleteStore for MemoryStore { .and_then(|o| o.remove(&(*id, generation))) .map(|_| ())) } + async fn cleanup_stages( + &self, + _space: &SpaceId, + _id: &Hash, + _generation: i64, + ) -> Result<(), Self::Error> { + // Memory staging has no filesystem artifact after the writer exits. + Ok(()) + } } #[async_trait] diff --git a/tinycloud-core/src/storage/mod.rs b/tinycloud-core/src/storage/mod.rs index d29a31b1..8bbda7c6 100644 --- a/tinycloud-core/src/storage/mod.rs +++ b/tinycloud-core/src/storage/mod.rs @@ -213,6 +213,16 @@ pub trait ImmutableDeleteStore: Send + Sync { id: &Hash, generation: i64, ) -> Result, Self::Error>; + + /// Reclaim incomplete, unreferenced staging files without removing the + /// published blob. Called only after the orphan fence found no active + /// attempts; storage backends without persistent stages have nothing to do. + async fn cleanup_stages( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result<(), Self::Error>; } #[async_trait] @@ -321,6 +331,14 @@ where ) -> Result, Self::Error> { (**self).remove(space, id, generation).await } + async fn cleanup_stages( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result<(), Self::Error> { + (**self).cleanup_stages(space, id, generation).await + } } #[async_trait] diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index 5566471d..eb62a350 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -10,6 +10,8 @@ use futures::{ }; use pin_project::pin_project; use serde::{Deserialize, Serialize}; +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; use std::{ collections::{hash_map::Entry, HashMap}, fmt::Write as _, @@ -45,6 +47,10 @@ pub struct FileSystemStore { before_memory_write: TestPause, #[cfg(test)] after_memory_contains: TestPause, + #[cfg(test)] + fail_stage_cleanup: Arc, + #[cfg(test)] + after_stage_unlink: TestPause, } impl FileSystemStore { @@ -60,6 +66,10 @@ impl FileSystemStore { before_memory_write: Arc::new(Mutex::new(None)), #[cfg(test)] after_memory_contains: Arc::new(Mutex::new(None)), + #[cfg(test)] + fail_stage_cleanup: Arc::new(AtomicBool::new(false)), + #[cfg(test)] + after_stage_unlink: Arc::new(Mutex::new(None)), }) } @@ -136,6 +146,11 @@ impl FileSystemStore { removed = removed .checked_add(size) .ok_or_else(|| IoError::other("staged blob byte count overflow"))?; + #[cfg(test)] + if let Some((entered, release)) = self.after_stage_unlink.lock().await.take() { + entered.notify_one(); + release.notified().await; + } if refund_each { self.decrement_size(space, size).await; } @@ -565,6 +580,36 @@ impl ImmutableDeleteStore for FileSystemStore { } Ok((removed_final.is_some() || staged != 0).then_some(())) } + async fn cleanup_stages( + &self, + space: &SpaceId, + id: &Hash, + generation: i64, + ) -> Result<(), Self::Error> { + let store = self.clone(); + let space = space.clone(); + let id = *id; + // The DB fence may time out or its caller may be cancelled. Finish + // each unlink and its quota refund together before releasing the + // physical lock; the retained DB attempt makes a retry idempotent. + tokio::spawn(async move { + let key = (space.clone(), id, generation); + let _operation = store.operations.lock(key.clone()).await; + #[cfg(test)] + if store.fail_stage_cleanup.load(Ordering::SeqCst) { + return Err(IoError::other("injected stage cleanup failure").into()); + } + let pending = store.pending.lock().await.contains_key(&key); + // This path is for a referenced final blob. Only incomplete + // stages are removed; never unlink a committed row's object. + store + .remove_stages(&space, &store.get_path(&space, &id, generation), !pending) + .await?; + Ok::<(), FileSystemStoreError>(()) + }) + .await + .map_err(IoError::other)? + } } #[async_trait] @@ -736,6 +781,218 @@ mod test { assert!(!stage_path.exists()); } + #[tokio::test] + async fn referenced_retry_collects_crashed_stage_without_deleting_committed_blob() { + referenced_retry_with_crashed_stage(false, false, false).await; + } + + #[tokio::test] + async fn referenced_retry_retains_stage_recovery_if_crash_attempt_expires_first() { + referenced_retry_with_crashed_stage(true, false, false).await; + } + + #[tokio::test] + async fn referenced_retry_retains_candidate_when_stage_cleanup_fails() { + referenced_retry_with_crashed_stage(false, true, false).await; + } + + #[tokio::test] + async fn referenced_retry_cancelled_cleanup_completes_stage_refund() { + referenced_retry_with_crashed_stage(false, false, true).await; + } + + async fn referenced_retry_with_crashed_stage( + collect_before_commit: bool, + fail_cleanup: bool, + cancel_cleanup: bool, + ) { + use tinycloud_core::sea_orm::{ + ConnectionTrait, Database, DbBackend, EntityTrait, PaginatorTrait, Statement, + TransactionTrait, + }; + use tinycloud_core::sea_orm_migration::MigratorTrait; + use tinycloud_core::{ + hash::hash, + kv_orphans, + migrations::Migrator, + models::{kv_orphan_attempt, kv_orphan_candidate}, + }; + + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + db.execute_unprepared("PRAGMA foreign_keys = OFF") + .await + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let cfg = FileSystemConfig::new(dir.path()); + let store = cfg.open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:referenced-stage".parse().unwrap(); + store.create(&space).await.unwrap(); + let data = b"complete same-hash retry after crash"; + let value = hash(data); + let blobs = [(space.clone(), value)]; + let tx = db.begin().await.unwrap(); + kv_orphans::register_candidates(&tx, &blobs, "crashed-A") + .await + .unwrap(); + tx.commit().await.unwrap(); + + let path = store.get_path(&space, &value, 0); + let staged = tempfile::Builder::new() + .prefix(&FileSystemStore::stage_prefix(&path)) + .tempfile_in(path.parent().unwrap()) + .unwrap(); + let stage_path = staged.path().to_path_buf(); + std::fs::write(&stage_path, b"partial").unwrap(); + staged.into_temp_path().keep().unwrap(); + drop(store); + let store = cfg.open().await.unwrap(); + assert_eq!(store.total_size(&space).await.unwrap(), Some(7)); + + let tx = db.begin().await.unwrap(); + let registered = kv_orphans::register_candidates(&tx, &blobs, "retry-B") + .await + .unwrap(); + tx.commit().await.unwrap(); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(data).await.unwrap(); + ImmutableWriteStore::::persist( + &store, + &space, + stage, + registered[0].2, + ) + .await + .unwrap(); + if collect_before_commit { + // A's DB-time claim is older than the five-minute grace; B's + // fresh claim still protects its in-flight upload. + db.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "UPDATE kv_orphan_attempt SET created_at = ? WHERE attempt_id = ?", + [ + (time::OffsetDateTime::now_utc() - time::Duration::minutes(6)).into(), + "crashed-A".into(), + ], + )) + .await + .unwrap(); + kv_orphans::collect_candidates(&db, &store, 256, kv_orphans::ORPHAN_GRACE) + .await + .unwrap(); + assert!(stage_path.exists()); + } + let tx = db.begin().await.unwrap(); + kv_orphans::lock_candidates(&tx, ®istered, "retry-B") + .await + .unwrap(); + tx.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "INSERT INTO kv_write (space,key,invocation,seq,epoch,epoch_seq,value,metadata,generation) + VALUES (?,'retry',?,1,?,1,?,'{}',?)", + [ + space.to_string().into(), + Vec::::from(hash(b"invocation")).into(), + Vec::::from(hash(b"epoch")).into(), + Vec::::from(value).into(), + registered[0].2.into(), + ], + )) + .await + .unwrap(); + kv_orphans::finish_candidates(&tx, ®istered, "retry-B") + .await + .unwrap(); + tx.commit().await.unwrap(); + + if fail_cleanup { + store.fail_stage_cleanup.store(true, Ordering::SeqCst); + assert!( + kv_orphans::collect_candidates(&db, &store, 256, time::Duration::ZERO) + .await + .is_err() + ); + assert!(stage_path.exists()); + assert_eq!( + kv_orphan_candidate::Entity::find() + .count(&db) + .await + .unwrap(), + 1 + ); + assert_eq!( + kv_orphan_attempt::Entity::find().count(&db).await.unwrap(), + 1 + ); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(7 + data.len() as u64) + ); + store.fail_stage_cleanup.store(false, Ordering::SeqCst); + kv_orphans::refund_failed(&db, &store, ®istered, "crashed-A") + .await + .unwrap(); + } else if cancel_cleanup { + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.after_stage_unlink.lock().await = Some((entered.clone(), release.clone())); + let collecting_db = db.clone(); + let collecting_store = store.clone(); + let collector = tokio::spawn(async move { + kv_orphans::collect_candidates( + &collecting_db, + &collecting_store, + 256, + time::Duration::ZERO, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + assert!(!stage_path.exists()); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(7 + data.len() as u64) + ); + collector.abort(); + assert!(collector.await.unwrap_err().is_cancelled()); + release.notify_one(); + tokio::time::timeout( + std::time::Duration::from_secs(2), + kv_orphans::refund_failed(&db, &store, ®istered, "crashed-A"), + ) + .await + .unwrap() + .unwrap(); + } else { + kv_orphans::collect_candidates(&db, &store, 256, time::Duration::ZERO) + .await + .unwrap(); + } + assert_eq!( + store.read_to_vec(&space, &value, 0).await.unwrap(), + Some(data.to_vec()), + "GC must preserve the committed retry" + ); + assert!(!stage_path.exists(), "the crashed stage is unreferenced"); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(data.len() as u64) + ); + assert_eq!( + kv_orphan_candidate::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); + assert_eq!( + kv_orphan_attempt::Entity::find().count(&db).await.unwrap(), + 0 + ); + } + #[tokio::test] async fn failed_file_creation_refunds_reserved_quota() { let dir = tempfile::tempdir().unwrap(); diff --git a/tinycloud-node-server/src/storage/s3.rs b/tinycloud-node-server/src/storage/s3.rs index 0fe499e5..a70d4e5f 100644 --- a/tinycloud-node-server/src/storage/s3.rs +++ b/tinycloud-node-server/src/storage/s3.rs @@ -615,6 +615,15 @@ impl ImmutableDeleteStore for S3BlockStore { Err(e) => Err(S3Error::from(e).into()), } } + async fn cleanup_stages( + &self, + _space: &SpaceId, + _id: &Hash, + _generation: i64, + ) -> Result<(), Self::Error> { + // Staging is not in the bucket; an incomplete S3 PUT has no visible key. + Ok(()) + } } #[async_trait] From 9e6fb86929bed31838689ea7a698a05c79543371 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 02:48:20 +0000 Subject: [PATCH 08/17] fix: drain expired blob claims without spinning collector --- CHANGELOG.md | 4 +- tinycloud-core/src/kv_orphans.rs | 248 ++++++++++++++++++++++++++++--- tinycloud-node-server/src/lib.rs | 19 +-- 3 files changed, 238 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9cda064..40dcc316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## [Unreleased] -- KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). +- KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace when no same-hash writer remains active, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. - Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. + Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans, and each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index c831cdd3..e37d3679 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -528,8 +528,8 @@ async fn lock_fence( } } -/// Drop one durable attempt and, if it is the last unreferenced claim, commit -/// the deleting fence and its byte entitlement before external I/O. +/// Retire a failed attempt or an expired GC group under the hash fence. GC +/// keeps an expired cleanup claim only while a genuinely live sibling exists. async fn prepare_deletion( conn: &C, store: &S, @@ -537,7 +537,7 @@ async fn prepare_deletion( value: Hash, generation: i64, attempt_id: &str, - skip_locked: bool, + gc_cutoff: Option, ) -> Result::Error>> where C: ConnectionTrait + TransactionTrait, @@ -553,7 +553,7 @@ where let now = database_now(conn).await?; let tx = conn.begin().await?; let key = SpaceIdWrap(space.clone()); - let Some(fence) = lock_fence(&tx, &key, value, generation, skip_locked).await? else { + let Some(fence) = lock_fence(&tx, &key, value, generation, gc_cutoff.is_some()).await? else { tx.commit().await?; return Ok(false); }; @@ -561,26 +561,53 @@ where tx.commit().await?; return Ok(false); } - let removed = attempt::Entity::delete_many() + let mut attempts = attempt::Entity::delete_many() .filter(attempt::Column::Space.eq(key.clone())) .filter(attempt::Column::Value.eq(value)) - .filter(attempt::Column::AttemptId.eq(attempt_id)) - .filter(attempt::Column::Generation.eq(generation)) - .exec(&tx) - .await?; + .filter(attempt::Column::Generation.eq(generation)); + if let Some(cutoff) = gc_cutoff { + // The fence excludes new registrations. If a live sibling exists, + // retain this expired claim as the stage-cleanup obligation, but move + // its next eligibility just past this pass's DB-time cutoff so a full + // blocked page cannot hide unrelated orphans. + let live = attempt::Entity::find() + .filter(attempt::Column::Space.eq(key.clone())) + .filter(attempt::Column::Value.eq(value)) + .filter(attempt::Column::Generation.eq(generation)) + .filter(attempt::Column::AttemptId.ne(attempt_id)) + .filter(attempt::Column::CreatedAt.gt(cutoff)) + .one(&tx) + .await? + .is_some(); + if live { + attempt::Entity::update_many() + .col_expr( + attempt::Column::CreatedAt, + Expr::value(cutoff + Duration::seconds(59)), + ) + .filter(attempt::Column::Space.eq(key)) + .filter(attempt::Column::Value.eq(value)) + .filter(attempt::Column::Generation.eq(generation)) + .filter(attempt::Column::AttemptId.eq(attempt_id)) + .filter(attempt::Column::CreatedAt.lte(cutoff)) + .exec(&tx) + .await?; + tx.commit().await?; + return Ok(false); + } + // All siblings are expired. Drain them together under this fence; + // deleting one at a time would make two crashed uploads immortal. + attempts = attempts.filter(attempt::Column::CreatedAt.lte(cutoff)); + } else { + attempts = attempts.filter(attempt::Column::AttemptId.eq(attempt_id)); + } + let removed = attempts.exec(&tx).await?; if removed.rows_affected == 0 { tx.commit().await?; return Ok(false); } - if has_attempts(&tx, &key, value, generation).await? { - if skip_locked { - // GC must retain the expired claim while another upload is - // in flight. Its crashed stage is still owed cleanup even if the - // other writer commits and would otherwise retire the fence. - tx.rollback().await?; - } else { - tx.commit().await?; - } + if gc_cutoff.is_none() && has_attempts(&tx, &key, value, generation).await? { + tx.commit().await?; return Ok(false); } if has_references(&tx, &key, value, generation).await? { @@ -751,7 +778,7 @@ where S: ImmutableDeleteStore + ImmutableReadStore, { for (space, value, generation) in blobs { - if prepare_deletion(conn, store, space, *value, *generation, attempt_id, false).await? { + if prepare_deletion(conn, store, space, *value, *generation, attempt_id, None).await? { complete_deletion(conn, store, space, *value, *generation).await?; } } @@ -831,7 +858,7 @@ where row.value, row.generation, &row.attempt_id, - true, + Some(cutoff), ) .await { @@ -865,6 +892,32 @@ where Ok((removed, scanned)) } +/// Drain due work for one scheduled collector tick. A full page can contain +/// only temporarily retained claims; never spin indefinitely on it or keep a +/// database connection busy while starving later candidates. +pub async fn drain_candidates_for_interval( + conn: &C, + store: &S, + min_age: Duration, +) -> Result::Error>> +where + C: ConnectionTrait + TransactionTrait, + S: ImmutableDeleteStore + ImmutableReadStore, +{ + const PAGE_SIZE: usize = 256; + const MAX_PAGES_PER_INTERVAL: usize = 4; + let mut removed = 0; + for _ in 0..MAX_PAGES_PER_INTERVAL { + let (deleted, scanned) = + collect_candidates_with_progress(conn, store, PAGE_SIZE as u64, min_age).await?; + removed += deleted; + if scanned < PAGE_SIZE { + break; + } + } + Ok(removed) +} + #[cfg(test)] mod tests { use super::*; @@ -1300,6 +1353,114 @@ mod tests { .is_none() ); } + #[tokio::test] + async fn expired_same_hash_attempts_drain_as_one_unreferenced_group() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"two crashed uploads").await; + let blob = [(space.clone(), value)]; + register(&db, &blob, "crashed-a").await; + register(&db, &blob, "crashed-b").await; + age(&db, &space, value).await; + + assert_eq!( + collect_candidates(&db, &store, 256, ORPHAN_GRACE) + .await + .unwrap(), + 1 + ); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert_eq!( + kv_orphan_attempt::Entity::find() + .all(&db) + .await + .unwrap() + .len(), + 0 + ); + assert_eq!( + kv_orphan_candidate::Entity::find() + .all(&db) + .await + .unwrap() + .len(), + 0 + ); + } + + #[tokio::test] + async fn expired_same_hash_attempts_drain_without_removing_committed_reference() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"referenced retry").await; + let blob = [(space.clone(), value)]; + register(&db, &blob, "crashed-a").await; + register(&db, &blob, "crashed-b").await; + age(&db, &space, value).await; + reference(&db, "kv_write", &space, value, 0).await; + + assert_eq!( + collect_candidates(&db, &store, 256, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &value, 0).await.unwrap()); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(b"referenced retry".len() as u64) + ); + assert_eq!( + kv_orphan_attempt::Entity::find() + .all(&db) + .await + .unwrap() + .len(), + 0 + ); + assert_eq!( + kv_orphan_candidate::Entity::find() + .all(&db) + .await + .unwrap() + .len(), + 0 + ); + } + + #[tokio::test] + async fn full_page_retained_for_live_writer_does_not_hide_later_orphans() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let blocked = upload(&store, &space, b"blocked group").await; + let blocked_blob = [(space.clone(), blocked)]; + for index in 0..256 { + register(&db, &blocked_blob, &format!("crashed-{index}")).await; + } + age(&db, &space, blocked).await; + register(&db, &blocked_blob, "active-writer").await; + + let later = upload(&store, &space, b"independent later orphan").await; + register(&db, &[(space.clone(), later)], "later-crash").await; + age(&db, &space, later).await; + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(3), + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE), + ) + .await + .expect("one collector tick must terminate on a blocked full page") + .unwrap(), + 1, + "the retained page must not starve an independent later orphan" + ); + assert!(!store.contains(&space, &later, 0).await.unwrap()); + assert!(store.contains(&space, &blocked, 0).await.unwrap()); + } #[tokio::test] async fn replayed_registration_requires_own_attempt_without_duplicating_it() { @@ -1577,6 +1738,40 @@ mod tests { ); } + #[tokio::test] + async fn postgres_expired_same_hash_claims_drain_under_one_fence() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_expired_same_hash_claims_drain_under_one_fence", + ) + .await + else { + return; + }; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"expired-pg-group").await; + let blob = [(space.clone(), value)]; + register(&fixture.conn, &blob, "crashed-a").await; + register(&fixture.conn, &blob, "crashed-b").await; + age(&fixture.conn, &space, value).await; + assert_eq!( + collect_candidates(&fixture.conn, &store, 256, ORPHAN_GRACE) + .await + .unwrap(), + 1 + ); + assert_eq!( + kv_orphan_attempt::Entity::find() + .all(&fixture.conn) + .await + .unwrap() + .len(), + 0 + ); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + #[tokio::test] async fn postgres_failed_attempt_cannot_erase_another_durable_registration() { let Some(fixture) = crate::test_support::PostgresTestSchema::new( @@ -1846,6 +2041,19 @@ mod tests { ); refund_failed(&db, &store, &third, "c").await.unwrap(); assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + let expired = upload(&inner, &space, b"mysql-two-expired").await; + let expired_blob = [(space.clone(), expired)]; + register(&db, &expired_blob, "expired-a").await; + register(&db, &expired_blob, "expired-b").await; + age(&db, &space, expired).await; + assert_eq!( + collect_candidates(&db, &store, 256, ORPHAN_GRACE) + .await + .unwrap(), + 1 + ); + assert!(!inner.contains(&space, &expired, 0).await.unwrap()); + assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); db.close().await.unwrap(); admin .execute(Statement::from_string( diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 544fd2e0..88882245 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -789,17 +789,14 @@ pub async fn app_with_control( loop { tokio::select! { _ = interval.tick() => { - loop { - match tinycloud_core::kv_orphans::collect_candidates_with_progress( - &conn, &store, 256, tinycloud_core::kv_orphans::ORPHAN_GRACE, - ).await { - Ok((_, 256)) => continue, - Ok(_) => break, - Err(error) => { - ::tracing::warn!(%error, "KV orphan collection failed"); - break; - } - } + let result = tokio::select! { + result = tinycloud_core::kv_orphans::drain_candidates_for_interval( + &conn, &store, tinycloud_core::kv_orphans::ORPHAN_GRACE, + ) => result, + _ = &mut shutdown => break, + }; + if let Err(error) = result { + ::tracing::warn!(%error, "KV orphan collection failed"); } } _ = &mut shutdown => break, From 3ec223ae53f3ecc84c80e2a9eb96973228114034 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 03:28:26 +0000 Subject: [PATCH 09/17] fix: keep orphan registration age immutable across retries --- CHANGELOG.md | 2 +- tinycloud-core/src/kv_orphans.rs | 184 ++++++++++++++++-- .../src/models/kv_orphan_attempt.rs | 2 + 3 files changed, 173 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40dcc316..146bbe47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace when no same-hash writer remains active, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. - Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans, and each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. + Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans. Registration age stays immutable while separate retry eligibility delays blocked claims, so a deferred crash claim cannot masquerade as a live uploader across collectors. Each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index e37d3679..d64b01ba 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -4,8 +4,8 @@ use std::{borrow::Cow, error::Error as StdError}; use sea_orm::{ sea_query::{Expr, LockBehavior, LockType, OnConflict}, - ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, QueryFilter, - QueryOrder, QuerySelect, Statement, TransactionTrait, + ColumnTrait, Condition, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, + QueryFilter, QueryOrder, QuerySelect, Statement, TransactionTrait, }; use time::{Duration, OffsetDateTime}; use tinycloud_auth::resource::SpaceId; @@ -222,6 +222,7 @@ pub async fn register_candidates( attempt_id: sea_orm::ActiveValue::Set(attempt_id.to_owned()), generation: sea_orm::ActiveValue::Set(generation), created_at: sea_orm::ActiveValue::Set(now), + retry_after: sea_orm::ActiveValue::Set(None), }) .on_conflict( OnConflict::columns([ @@ -358,6 +359,14 @@ pub async fn finish_candidates( AND EXISTS(SELECT 1 FROM locked) AND (SELECT present FROM refs) RETURNING space, value + ), woken AS ( + UPDATE kv_orphan_attempt AS other + SET retry_after = NULL + WHERE other.space = $1 AND other.value = $2 + AND other.generation = $4 AND other.attempt_id <> $3 + AND other.retry_after IS NOT NULL + AND EXISTS(SELECT 1 FROM removed) + RETURNING 1 ), retired AS ( DELETE FROM kv_orphan_candidate AS fence USING removed @@ -375,6 +384,7 @@ pub async fn finish_candidates( ) SELECT (SELECT COUNT(*) FROM removed) AS removed, (SELECT present FROM refs) AS referenced, + (SELECT COUNT(*) FROM woken) AS woken, (SELECT COUNT(*) FROM retired) AS retired", [key.into(), (*value).into(), attempt_id.to_owned().into(), (*generation).into()], )) @@ -402,9 +412,9 @@ pub async fn finish_candidates( "KV blob attempt disappeared before commit".into(), )); } - if !has_attempts(tx, &key, *value, *generation).await? - && has_references(tx, &key, *value, *generation).await? - { + if has_attempts(tx, &key, *value, *generation).await? { + wake_attempts(tx, &key, *value, *generation, attempt_id).await?; + } else if has_references(tx, &key, *value, *generation).await? { candidate::Entity::delete_many() .filter(candidate::Column::Space.eq(key)) .filter(candidate::Column::Value.eq(*value)) @@ -416,6 +426,28 @@ pub async fn finish_candidates( Ok(()) } +/// A writer leaving makes older deferred stage claims eligible immediately. +/// Both the successful commit and the failure path call this under the fence; +/// registration age remains immutable throughout. +async fn wake_attempts( + conn: &C, + space: &SpaceIdWrap, + value: Hash, + generation: i64, + departing_attempt: &str, +) -> Result<(), DbErr> { + attempt::Entity::update_many() + .col_expr(attempt::Column::RetryAfter, Expr::cust("NULL")) + .filter(attempt::Column::Space.eq(space.clone())) + .filter(attempt::Column::Value.eq(value)) + .filter(attempt::Column::Generation.eq(generation)) + .filter(attempt::Column::AttemptId.ne(departing_attempt)) + .filter(attempt::Column::RetryAfter.is_not_null()) + .exec(conn) + .await?; + Ok(()) +} + #[derive(Debug, thiserror::Error)] pub enum CollectError { #[error("orphan candidate database operation failed: {0}")] @@ -566,10 +598,10 @@ where .filter(attempt::Column::Value.eq(value)) .filter(attempt::Column::Generation.eq(generation)); if let Some(cutoff) = gc_cutoff { - // The fence excludes new registrations. If a live sibling exists, - // retain this expired claim as the stage-cleanup obligation, but move - // its next eligibility just past this pass's DB-time cutoff so a full - // blocked page cannot hide unrelated orphans. + // The fence excludes new registrations. A live sibling retains this + // expired claim's stage obligation, but its retry eligibility must be + // separate from immutable upload age: other collectors must never + // mistake a deferred crashed attempt for a new live writer. let live = attempt::Entity::find() .filter(attempt::Column::Space.eq(key.clone())) .filter(attempt::Column::Value.eq(value)) @@ -582,8 +614,8 @@ where if live { attempt::Entity::update_many() .col_expr( - attempt::Column::CreatedAt, - Expr::value(cutoff + Duration::seconds(59)), + attempt::Column::RetryAfter, + Expr::value(now + Duration::seconds(59)), ) .filter(attempt::Column::Space.eq(key)) .filter(attempt::Column::Value.eq(value)) @@ -607,6 +639,7 @@ where return Ok(false); } if gc_cutoff.is_none() && has_attempts(&tx, &key, value, generation).await? { + wake_attempts(&tx, &key, value, generation, attempt_id).await?; tx.commit().await?; return Ok(false); } @@ -756,8 +789,8 @@ async fn defer_attempt( ) -> Result<(), DbErr> { attempt::Entity::update_many() .col_expr( - attempt::Column::CreatedAt, - Expr::value(database_now(conn).await?), + attempt::Column::RetryAfter, + Expr::value(database_now(conn).await? + Duration::seconds(59)), ) .filter(attempt::Column::Space.eq(SpaceIdWrap(space.clone()))) .filter(attempt::Column::Value.eq(value)) @@ -832,6 +865,11 @@ where .await?; let attempts = attempt::Entity::find() .filter(attempt::Column::CreatedAt.lte(cutoff)) + .filter( + Condition::any() + .add(attempt::Column::RetryAfter.is_null()) + .add(attempt::Column::RetryAfter.lte(now)), + ) .order_by_asc(attempt::Column::CreatedAt) .order_by_asc(attempt::Column::Space) .order_by_asc(attempt::Column::Value) @@ -1431,6 +1469,123 @@ mod tests { ); } + #[tokio::test] + async fn staggered_collectors_drain_crashed_claims_after_last_live_writer_exits() { + let directory = tempfile::tempdir().unwrap(); + let url = format!( + "sqlite://{}?mode=rwc", + directory + .path() + .join("staggered-collectors.sqlite") + .display() + ); + let first = Database::connect(ConnectOptions::new(url.clone())) + .await + .unwrap(); + crate::migrations::Migrator::up(&first, None).await.unwrap(); + let second = Database::connect(ConnectOptions::new(url)).await.unwrap(); + for db in [&first, &second] { + db.execute(Statement::from_string( + DbBackend::Sqlite, + "PRAGMA foreign_keys = OFF", + )) + .await + .unwrap(); + } + + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"two stale claims").await; + let blob = [(space.clone(), value)]; + register(&first, &blob, "crashed-a").await; + register(&first, &blob, "crashed-b").await; + age(&first, &space, value).await; + let live = register(&first, &blob, "live-c").await; + + // First collector defers one old claim while C is still uploading. + assert_eq!( + collect_candidates(&first, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + refund_failed(&first, &store, &live, "live-c") + .await + .unwrap(); + assert!(store.contains(&space, &value, 0).await.unwrap()); + + // A second collector must not mistake A's retry delay for a live + // writer; with no real uploader left, the group and bytes drain now. + assert_eq!( + collect_candidates(&second, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 1 + ); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert!(kv_orphan_attempt::Entity::find() + .all(&second) + .await + .unwrap() + .is_empty()); + } + + #[tokio::test] + async fn failed_gc_read_does_not_make_crashed_sibling_look_live() { + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let value = upload(&inner, &space, b"failed first collector read").await; + let blob = [(space.clone(), value)]; + register(&db, &blob, "crashed-a").await; + register(&db, &blob, "crashed-b").await; + age(&db, &space, value).await; + + store.fail_read.store(true, Ordering::SeqCst); + assert!(collect_candidates(&db, &store, 1, ORPHAN_GRACE) + .await + .is_err()); + assert_eq!( + collect_candidates(&db, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 1, + "a failed read must not turn its crashed claim into a live writer" + ); + assert!(!inner.contains(&space, &value, 0).await.unwrap()); + assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn last_failed_writer_wakes_deferred_crash_cleanup() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let value = upload(&store, &space, b"last failed writer").await; + let blob = [(space.clone(), value)]; + register(&db, &blob, "crashed-a").await; + age(&db, &space, value).await; + let live = register(&db, &blob, "live-b").await; + assert_eq!( + collect_candidates(&db, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + refund_failed(&db, &store, &live, "live-b").await.unwrap(); + assert_eq!( + collect_candidates(&db, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 1, + "the final writer leaving must expose the old claim immediately" + ); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + #[tokio::test] async fn full_page_retained_for_live_writer_does_not_hide_later_orphans() { let db = database().await; @@ -1993,7 +2148,8 @@ mod tests { "CREATE TABLE kv_orphan_attempt ( space VARCHAR(191) NOT NULL, value VARBINARY(34) NOT NULL, attempt_id VARCHAR(32) NOT NULL, generation BIGINT NOT NULL, - created_at DATETIME(6) NOT NULL, PRIMARY KEY (space, value, attempt_id))", + created_at DATETIME(6) NOT NULL, retry_after DATETIME(6) NULL, + PRIMARY KEY (space, value, attempt_id))", "CREATE TABLE kv_write (space VARCHAR(191), value VARBINARY(34), generation BIGINT NOT NULL DEFAULT 0, seq BIGINT NOT NULL, INDEX (space, value))", "CREATE TABLE current_kv (space VARCHAR(191), value VARBINARY(34), diff --git a/tinycloud-core/src/models/kv_orphan_attempt.rs b/tinycloud-core/src/models/kv_orphan_attempt.rs index e0d33b19..46912aff 100644 --- a/tinycloud-core/src/models/kv_orphan_attempt.rs +++ b/tinycloud-core/src/models/kv_orphan_attempt.rs @@ -14,6 +14,8 @@ pub struct Model { pub attempt_id: String, pub generation: i64, pub created_at: OffsetDateTime, + /// Collector backoff only; `created_at` remains the immutable upload age. + pub retry_after: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] From b5febbc275edf3bf83ae4f7f424276c95d0046fe Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 04:06:19 +0000 Subject: [PATCH 10/17] Fix filesystem stage quota recovery after failed retries --- CHANGELOG.md | 1 + .../src/storage/file_system.rs | 464 ++++++++++++++---- 2 files changed, 378 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 146bbe47..d0c55ae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - KV puts persist content-addressed blobs before taking the per-space sequence lock, while committed `kv/sync` cursors remain ordered. A short preparatory transaction registers one durable attempt per write; ordinary uploads hold no pooled DB connection, unlike the previous write path. Protected legacy meeting writes remain the documented exception: their pre-upload freeze barrier waits for commit or abort and fails after 30 seconds if stalled (TC-1260). Failed attempts transition their last unreferenced hash fence to `deleting`; a new writer cannot deduplicate or upload until confirmed physical absence removes that fence. Known failures refund once after deletion; a 60-second collector reclaims crash leftovers after a five-minute DB-time grace when no same-hash writer remains active, including lost DELETE responses. S3 startup reconstructs pending refunds from the same physical listing as its quota count. Quota remains conservative and per process, not a durable global ledger (TC-735). Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans. Registration age stays immutable while separate retry eligibility delays blocked claims, so a deferred crash claim cannot masquerade as a live uploader across collectors. Each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. + Boot quota reconstruction tracks stage-file charges from the same directory listing as the total; a failed retry and an older crash-leftover stage each refund their own charge after confirmed removal. Known filesystem-stage failures refund their reservation immediately after the unpublished file is removed, and cancellation of a delete cannot interrupt its physical unlink and quota settlement. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index eb62a350..a5ad9069 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -3,7 +3,7 @@ use super::TestPause; use super::{size::SpaceSizes, BlobGenerationKey, BlobOperationLocks}; use core::pin::Pin; use futures::{ - future::{Either as AsyncEither, TryFutureExt}, + future::Either as AsyncEither, io::{AsyncWrite, AsyncWriteExt}, stream::TryStreamExt, task::{Context, Poll}, @@ -42,6 +42,9 @@ pub struct FileSystemStore { // or failed publish leaves a conservative charge until cleanup confirms // that neither the stage nor the final object remains. pending: Arc>>, + // Paths and charges from the same boot listing as `sizes`. A later upload + // reservation for the same hash is a separate charge, not a replacement. + boot_stages: Arc>>, operations: BlobOperationLocks, #[cfg(test)] before_memory_write: TestPause, @@ -55,12 +58,13 @@ pub struct FileSystemStore { impl FileSystemStore { async fn new(path: PathBuf) -> Result { - // get the size of the directory - let sizes = store_sizes(&path).await?.into(); + // Count each stage and its enclosing space from one listing snapshot. + let (sizes, boot_stages) = store_sizes(&path).await?; Ok(Self { path, - sizes, + sizes: sizes.into(), pending: Arc::new(Mutex::new(HashMap::new())), + boot_stages: Arc::new(Mutex::new(boot_stages)), operations: BlobOperationLocks::default(), #[cfg(test)] before_memory_write: Arc::new(Mutex::new(None)), @@ -106,6 +110,31 @@ impl FileSystemStore { .remove(&(space.clone(), hash, generation)); } + async fn refund_pending(&self, space: &SpaceId, hash: Hash, generation: i64) { + let reserved = self + .pending + .lock() + .await + .remove(&(space.clone(), hash, generation)) + .unwrap_or(0); + if reserved != 0 { + self.decrement_size(space, reserved).await; + } + } + + async fn discard_failed_stage( + &self, + space: &SpaceId, + hash: Hash, + generation: i64, + path: tempfile::TempPath, + ) -> Result<(), IoError> { + // Do not refund a reservation until its unpublished bytes are gone. + path.close()?; + self.refund_pending(space, hash, generation).await; + Ok(()) + } + fn stage_prefix(path: &Path) -> String { format!( ".tc-stage-{}-", @@ -123,17 +152,18 @@ impl FileSystemStore { space: &SpaceId, path: &Path, refund_each: bool, - ) -> Result { + ) -> Result<(u64, u64), IoError> { let Some(parent) = path.parent() else { - return Ok(0); + return Ok((0, 0)); }; let mut entries = match tokio::fs::read_dir(parent).await { Ok(entries) => entries, - Err(error) if error.kind() == ErrorKind::NotFound => return Ok(0), + Err(error) if error.kind() == ErrorKind::NotFound => return Ok((0, 0)), Err(error) => return Err(error), }; let prefix = Self::stage_prefix(path); let mut removed = 0u64; + let mut boot_refunded = 0u64; while let Some(entry) = entries.next_entry().await? { if !entry.file_name().to_string_lossy().starts_with(&prefix) || !entry.file_type().await?.is_file() @@ -141,7 +171,8 @@ impl FileSystemStore { continue; } let size = entry.metadata().await?.len(); - match remove_file(entry.path()).await { + let stage_path = entry.path(); + match remove_file(&stage_path).await { Ok(()) => { removed = removed .checked_add(size) @@ -151,15 +182,32 @@ impl FileSystemStore { entered.notify_one(); release.notified().await; } - if refund_each { - self.decrement_size(space, size).await; + let counted_at_boot = self + .boot_stages + .lock() + .await + .remove(&stage_path) + .unwrap_or(0); + boot_refunded = boot_refunded + .checked_add(counted_at_boot) + .ok_or_else(|| IoError::other("boot stage byte count overflow"))?; + if counted_at_boot != 0 || refund_each { + self.decrement_size( + space, + if counted_at_boot != 0 { + counted_at_boot + } else { + size + }, + ) + .await; } } Err(error) if error.kind() == ErrorKind::NotFound => {} Err(error) => return Err(error), } } - Ok(removed) + Ok((removed, boot_refunded)) } async fn persist_temp( @@ -180,9 +228,17 @@ impl FileSystemStore { let size = file.size().await?; self.reserve(space, hash, generation, size).await; let (_, path) = file.into_inner(); - path.persist(self.get_path(space, &hash, generation))?; - self.settle(space, hash, generation).await; - Ok(()) + match path.persist(self.get_path(space, &hash, generation)) { + Ok(()) => { + self.settle(space, hash, generation).await; + Ok(()) + } + Err(PathPersistError { error, path }) => { + self.discard_failed_stage(space, hash, generation, path) + .await?; + Err(error.into()) + } + } } async fn persist_memory( @@ -212,9 +268,16 @@ impl FileSystemStore { let target = self.get_path(space, &hash, generation); // Stage on the target filesystem so the final rename is atomic and // never depends on a cross-device copy. - let staged = tempfile::Builder::new() + let staged = match tempfile::Builder::new() .prefix(&Self::stage_prefix(&target)) - .tempfile_in(target.parent().expect("blob path has a parent"))?; + .tempfile_in(target.parent().expect("blob path has a parent")) + { + Ok(staged) => staged, + Err(error) => { + self.refund_pending(space, hash, generation).await; + return Err(error.into()); + } + }; let (file, path) = staged.into_parts(); #[cfg(test)] let pause = self.before_memory_write.lock().await.take(); @@ -224,12 +287,28 @@ impl FileSystemStore { release.notified().await; } let mut writer = File::from_std(file).compat(); - writer.write_all(&bytes).await?; - writer.flush().await?; + let written = async { + writer.write_all(&bytes).await?; + writer.flush().await + } + .await; drop(writer); - path.persist(target)?; - self.settle(space, hash, generation).await; - Ok(()) + if let Err(error) = written { + self.discard_failed_stage(space, hash, generation, path) + .await?; + return Err(error.into()); + } + match path.persist(target) { + Ok(()) => { + self.settle(space, hash, generation).await; + Ok(()) + } + Err(PathPersistError { error, path }) => { + self.discard_failed_stage(space, hash, generation, path) + .await?; + Err(error.into()) + } + } } } @@ -360,49 +439,72 @@ impl StoreSize for FileSystemStore { } // get the sum size of all files in this directory (recurse into subdirectories with space ID names) -async fn store_sizes>(path: &P) -> Result, IoError> { +async fn store_sizes>( + path: &P, +) -> Result<(HashMap, HashMap), IoError> { ReadDirStream::new(tokio::fs::read_dir(path).await?) // for every entry in the store dir - .try_fold(HashMap::new(), |mut acc, entry| async move { - // if its a directory and the suffix is a valid string - if let (true, Ok(ref suffix)) = ( - entry.metadata().await?.is_dir(), - entry.file_name().into_string(), - ) { - let mut ds = ReadDirStream::new(tokio::fs::read_dir(entry.path()).await?); - let did: DIDBuf = ["did:", suffix.as_str()].concat().parse().map_err(|_| { - IoError::new(ErrorKind::InvalidData, format!("Invalid DID: {suffix}")) - })?; - // go through each suffix directory - while let Some(entry) = ds.try_next().await? { - // for each entry in the suffix directory - // if its a directory and the name is a valid string - if let (true, Ok(name)) = ( - entry.metadata().await?.is_dir(), - entry.file_name().into_string(), - ) { - // get the space ID from suffix and name - let space = - SpaceId::new(did.clone(), name.try_into().map_err(IoError::other)?); - let size = space_size(&entry.path()).await?; - acc.insert(space, size); + .try_fold( + (HashMap::new(), HashMap::new()), + |(mut acc, mut stages), entry| async move { + // if its a directory and the suffix is a valid string + if let (true, Ok(ref suffix)) = ( + entry.metadata().await?.is_dir(), + entry.file_name().into_string(), + ) { + let mut ds = ReadDirStream::new(tokio::fs::read_dir(entry.path()).await?); + let did: DIDBuf = ["did:", suffix.as_str()].concat().parse().map_err(|_| { + IoError::new(ErrorKind::InvalidData, format!("Invalid DID: {suffix}")) + })?; + // go through each suffix directory + while let Some(entry) = ds.try_next().await? { + // for each entry in the suffix directory + // if its a directory and the name is a valid string + if let (true, Ok(name)) = ( + entry.metadata().await?.is_dir(), + entry.file_name().into_string(), + ) { + // get the space ID from suffix and name + let space = + SpaceId::new(did.clone(), name.try_into().map_err(IoError::other)?); + let (size, space_stages) = space_size(&entry.path()).await?; + acc.insert(space, size); + stages.extend(space_stages); + } } - } - }; - Ok(acc) - }) + }; + Ok((acc, stages)) + }, + ) .await } -async fn space_size>(path: &P) -> Result { - // get the sum size of all files in this directory (do not recurse into subdirectories) +async fn space_size>(path: &P) -> Result<(u64, HashMap), IoError> { + // This one directory listing establishes both the charged total and the + // exact incomplete stages which contributed to that total at boot. ReadDirStream::new(tokio::fs::read_dir(path).await?) - .try_fold(0, |acc, entry| async move { - entry - .metadata() - .map_ok(|m| if m.is_dir() { acc } else { acc + m.len() }) - .await - }) + .try_fold( + (0u64, HashMap::new()), + |(acc, mut stages), entry| async move { + let metadata = entry.metadata().await?; + if metadata.is_dir() { + return Ok((acc, stages)); + } + let size = metadata.len(); + if metadata.is_file() + && entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(".tc-stage-")) + { + stages.insert(entry.path(), size); + } + let total = acc + .checked_add(size) + .ok_or_else(|| IoError::other("space byte count overflow"))?; + Ok((total, stages)) + }, + ) .await } @@ -547,38 +649,51 @@ impl ImmutableDeleteStore for FileSystemStore { id: &Hash, generation: i64, ) -> Result, Self::Error> { - let key = (space.clone(), *id, generation); - let _operation = self.operations.lock(key.clone()).await; - let path = self.get_path(space, id, generation); - let pending = self.pending.lock().await.contains_key(&key); - // A crash can leave a named, incomplete memory stage. Boot counted - // its actual bytes; a live reservation instead counted its full size. - let staged = self.remove_stages(space, &path, !pending).await?; - let final_size = match metadata(&path).await { - Ok(meta) => Some(meta.len()), - Err(error) if error.kind() == ErrorKind::NotFound => None, - Err(error) => return Err(error.into()), - }; - let removed_final = if let Some(size) = final_size { - match remove_file(&path).await { - Ok(()) => Some(size), + let store = self.clone(); + let space = space.clone(); + let id = *id; + // Finish unlink and quota settlement under the same physical lock + // even if the request or collector is cancelled after unlink. + tokio::spawn(async move { + let key = (space.clone(), id, generation); + let _operation = store.operations.lock(key.clone()).await; + let path = store.get_path(&space, &id, generation); + let pending = store.pending.lock().await.contains_key(&key); + // Boot stages and a new reservation are independently charged. + let (staged, boot_refunded) = store.remove_stages(&space, &path, !pending).await?; + let final_size = match metadata(&path).await { + Ok(meta) => Some(meta.len()), Err(error) if error.kind() == ErrorKind::NotFound => None, Err(error) => return Err(error.into()), + }; + let removed_final = if let Some(size) = final_size { + match remove_file(&path).await { + Ok(()) => Some(size), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + } + } else { + None + }; + if pending { + let reserved = store.pending.lock().await.remove(&key).unwrap_or(0); + store + .decrement_size( + &space, + reserved.max( + staged + .saturating_sub(boot_refunded) + .saturating_add(removed_final.unwrap_or(0)), + ), + ) + .await; + } else if let Some(size) = removed_final { + store.decrement_size(&space, size).await; } - } else { - None - }; - if pending { - let reserved = self.pending.lock().await.remove(&key).unwrap_or(0); - self.decrement_size( - space, - reserved.max(staged.saturating_add(removed_final.unwrap_or(0))), - ) - .await; - } else if let Some(size) = removed_final { - self.decrement_size(space, size).await; - } - Ok((removed_final.is_some() || staged != 0).then_some(())) + Ok::<_, FileSystemStoreError>((removed_final.is_some() || staged != 0).then_some(())) + }) + .await + .map_err(IoError::other)? } async fn cleanup_stages( &self, @@ -605,6 +720,9 @@ impl ImmutableDeleteStore for FileSystemStore { store .remove_stages(&space, &store.get_path(&space, &id, generation), !pending) .await?; + if pending { + store.refund_pending(&space, id, generation).await; + } Ok::<(), FileSystemStoreError>(()) }) .await @@ -781,6 +899,162 @@ mod test { assert!(!stage_path.exists()); } + #[tokio::test] + async fn cancelled_delete_finishes_boot_stage_refund_before_retry() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:cancel-stage".parse().unwrap(); + store.create(&space).await.unwrap(); + let value = tinycloud_core::hash::hash(b"cancelled stage delete"); + let target = store.get_path(&space, &value, 0); + let mut stage = tempfile::Builder::new() + .prefix(&FileSystemStore::stage_prefix(&target)) + .tempfile_in(target.parent().unwrap()) + .unwrap(); + std::io::Write::write_all(&mut stage, b"partial").unwrap(); + let stage_path = stage.path().to_path_buf(); + stage.into_temp_path().keep().unwrap(); + drop(store); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + assert_eq!(store.total_size(&space).await.unwrap(), Some(7)); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.after_stage_unlink.lock().await = Some((entered.clone(), release.clone())); + let deleting_store = store.clone(); + let deleting_space = space.clone(); + let deletion = + tokio::spawn(async move { deleting_store.remove(&deleting_space, &value, 0).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + assert!(!stage_path.exists()); + deletion.abort(); + assert!(deletion.await.unwrap_err().is_cancelled()); + release.notify_one(); + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(2), + store.remove(&space, &value, 0) + ) + .await + .unwrap() + .unwrap(), + None + ); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn boot_counted_stage_and_failed_retry_refund_both_charges_once() { + use tinycloud_core::sea_orm::{ + ConnectionTrait, Database, DbBackend, Statement, TransactionTrait, + }; + use tinycloud_core::sea_orm_migration::MigratorTrait; + use tinycloud_core::{hash::hash, kv_orphans, migrations::Migrator}; + + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + db.execute_unprepared("PRAGMA foreign_keys = OFF") + .await + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:stage-and-reservation".parse().unwrap(); + store.create(&space).await.unwrap(); + let bytes = b"retrying the same hash after a crash"; + let value = hash(bytes); + let blob = [(space.clone(), value)]; + let tx = db.begin().await.unwrap(); + kv_orphans::register_candidates(&tx, &blob, "crashed-a") + .await + .unwrap(); + tx.commit().await.unwrap(); + + let target = store.get_path(&space, &value, 0); + let mut stage = tempfile::Builder::new() + .prefix(&FileSystemStore::stage_prefix(&target)) + .tempfile_in(target.parent().unwrap()) + .unwrap(); + std::io::Write::write_all(&mut stage, b"partial").unwrap(); + let boot_stage = stage.path().to_path_buf(); + stage.into_temp_path().keep().unwrap(); + drop(store); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + assert_eq!(store.total_size(&space).await.unwrap(), Some(7)); + + let tx = db.begin().await.unwrap(); + let failed = kv_orphans::register_candidates(&tx, &blob, "failed-b") + .await + .unwrap(); + tx.commit().await.unwrap(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *store.before_memory_write.lock().await = Some((entered.clone(), release)); + let mut retry = HashBuffer::new(Vec::new()); + retry.write_all(bytes).await.unwrap(); + let uploading_store = store.clone(); + let uploading_space = space.clone(); + let uploading = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &uploading_store, + &uploading_space, + retry, + 0, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(7 + bytes.len() as u64) + ); + uploading.abort(); + assert!(uploading.await.unwrap_err().is_cancelled()); + kv_orphans::refund_failed(&db, &store, &failed, "failed-b") + .await + .unwrap(); + + let unrelated = b"other"; + let mut other_stage = HashBuffer::new(Vec::new()); + other_stage.write_all(unrelated).await.unwrap(); + let other = + ImmutableWriteStore::::persist(&store, &space, other_stage, 0) + .await + .unwrap(); + db.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "UPDATE kv_orphan_attempt SET created_at = ? WHERE attempt_id = ?", + [ + (time::OffsetDateTime::now_utc() - time::Duration::minutes(6)).into(), + "crashed-a".into(), + ], + )) + .await + .unwrap(); + assert_eq!( + kv_orphans::collect_candidates(&db, &store, 256, kv_orphans::ORPHAN_GRACE) + .await + .unwrap(), + 1 + ); + assert!(!boot_stage.exists()); + assert!(!store.contains(&space, &value, 0).await.unwrap()); + assert_eq!( + store.read_to_vec(&space, &other, 0).await.unwrap(), + Some(unrelated.to_vec()) + ); + assert_eq!(store.total_size(&space).await.unwrap(), Some(5)); + assert_eq!( + kv_orphans::collect_candidates(&db, &store, 256, kv_orphans::ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + assert_eq!(store.total_size(&space).await.unwrap(), Some(5)); + } + #[tokio::test] async fn referenced_retry_collects_crashed_stage_without_deleting_committed_blob() { referenced_retry_with_crashed_stage(false, false, false).await; @@ -1010,10 +1284,26 @@ mod test { .await .is_err() ); - assert_eq!(store.total_size(&space).await.unwrap(), Some(12)); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); assert!(!store.contains(&space, &hash, 0).await.unwrap()); assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + + let mut staged = HashBuffer::new( + TempFileSystemStage + .get_staging_buffer(&space) + .await + .unwrap(), + ); + staged.write_all(b"failed-stage").await.unwrap(); + let staged_hash = staged.hash(); + assert!( + ImmutableWriteStore::::persist(&store, &space, staged, 0) + .await + .is_err() + ); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + assert_eq!(store.remove(&space, &staged_hash, 0).await.unwrap(), None); } #[tokio::test] From 4cc4a4eb3cfcfb7cc845faf308281067cfed0fd9 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 04:40:49 +0000 Subject: [PATCH 11/17] Keep orphan collection fair across retries and quota updates atomic --- CHANGELOG.md | 1 + tinycloud-core/src/kv_orphans.rs | 315 +++++++++++++++--- .../m20261015_020000_kv_orphan_candidates.rs | 12 + .../src/storage/file_system.rs | 135 +++++++- tinycloud-node-server/src/storage/size.rs | 6 + 5 files changed, 404 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0c55ae8..78906a8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans. Registration age stays immutable while separate retry eligibility delays blocked claims, so a deferred crash claim cannot masquerade as a live uploader across collectors. Each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. Boot quota reconstruction tracks stage-file charges from the same directory listing as the total; a failed retry and an older crash-leftover stage each refund their own charge after confirmed removal. Known filesystem-stage failures refund their reservation immediately after the unpublished file is removed, and cancellation of a delete cannot interrupt its physical unlink and quota settlement. + Filesystem reservation and refund update pending charges atomically with `SpaceSizes` across cancellation. Each bounded collector page fairly admits never-retried and due-retry attempts through indexed queries, while a full deleting-fence page reserves capacity for expired attempts; persistent storage failures or live same-hash writers cannot starve unrelated orphans across ticks. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index d64b01ba..ba807063 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -4,8 +4,8 @@ use std::{borrow::Cow, error::Error as StdError}; use sea_orm::{ sea_query::{Expr, LockBehavior, LockType, OnConflict}, - ColumnTrait, Condition, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, - QueryFilter, QueryOrder, QuerySelect, Statement, TransactionTrait, + ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, QueryFilter, + QueryOrder, QuerySelect, Statement, TransactionTrait, }; use time::{Duration, OffsetDateTime}; use tinycloud_auth::resource::SpaceId; @@ -854,7 +854,7 @@ where let cutoff = now .checked_sub(min_age) .unwrap_or(OffsetDateTime::UNIX_EPOCH); - let deleting = candidate::Entity::find() + let mut deleting = candidate::Entity::find() .filter(candidate::Column::DeletingAt.is_not_null()) .filter(candidate::Column::DeletingAt.lte(now)) .order_by_asc(candidate::Column::DeletingAt) @@ -863,19 +863,52 @@ where .limit(limit) .all(conn) .await?; - let attempts = attempt::Entity::find() - .filter(attempt::Column::CreatedAt.lte(cutoff)) - .filter( - Condition::any() - .add(attempt::Column::RetryAfter.is_null()) - .add(attempt::Column::RetryAfter.lte(now)), - ) - .order_by_asc(attempt::Column::CreatedAt) - .order_by_asc(attempt::Column::Space) - .order_by_asc(attempt::Column::Value) - .limit(limit.saturating_sub(deleting.len() as u64)) - .all(conn) - .await?; + // A full repeatedly failing DELETE page must not exclude independent + // expired uploads. Give each queue half a page if both have due work. + let available = if deleting.len() as u64 == limit { + limit.div_ceil(2) + } else { + limit - deleting.len() as u64 + }; + let attempts = if available == 0 { + Vec::new() + } else { + // Query the two retry states separately so each bounded page can use + // its age or retry index. Re-reading an oldest failing page every + // minute must not hide an unrelated, never-retried orphan. + let mut never_retried = attempt::Entity::find() + .filter(attempt::Column::CreatedAt.lte(cutoff)) + .filter(attempt::Column::RetryAfter.is_null()) + .order_by_asc(attempt::Column::CreatedAt) + .order_by_asc(attempt::Column::Space) + .order_by_asc(attempt::Column::Value) + .limit(available) + .all(conn) + .await?; + let due_retries = attempt::Entity::find() + .filter(attempt::Column::CreatedAt.lte(cutoff)) + .filter(attempt::Column::RetryAfter.lte(now)) + .order_by_asc(attempt::Column::RetryAfter) + .order_by_asc(attempt::Column::CreatedAt) + .order_by_asc(attempt::Column::Space) + .order_by_asc(attempt::Column::Value) + .limit(available) + .all(conn) + .await?; + never_retried.extend(due_retries); + never_retried.sort_unstable_by(|left, right| { + left.retry_after + .unwrap_or(left.created_at) + .cmp(&right.retry_after.unwrap_or(right.created_at)) + .then_with(|| left.created_at.cmp(&right.created_at)) + .then_with(|| left.space.cmp(&right.space)) + .then_with(|| left.value.cmp(&right.value)) + .then_with(|| left.attempt_id.cmp(&right.attempt_id)) + }); + never_retried.truncate(available as usize); + never_retried + }; + deleting.truncate((limit as usize).saturating_sub(attempts.len())); let scanned = deleting.len() + attempts.len(); let mut removed = 0; let mut first_error = None; @@ -985,6 +1018,9 @@ mod tests { pause: Arc, lose_response: Arc, fail_read: Arc, + fail_hash: Arc>>, + fail_remove: Arc, + allow_remove: Arc>>, } impl InterruptedDelete { @@ -996,6 +1032,9 @@ mod tests { pause: Arc::new(AtomicBool::new(false)), lose_response: Arc::new(AtomicBool::new(false)), fail_read: Arc::new(AtomicBool::new(false)), + fail_hash: Arc::new(tokio::sync::Mutex::new(None)), + fail_remove: Arc::new(AtomicBool::new(false)), + allow_remove: Arc::new(tokio::sync::Mutex::new(None)), } } } @@ -1020,7 +1059,13 @@ mod tests { id: &Hash, generation: i64, ) -> Result>, Self::Error> { - if self.fail_read.swap(false, Ordering::SeqCst) { + if self + .fail_hash + .lock() + .await + .is_some_and(|value| value == *id) + || self.fail_read.swap(false, Ordering::SeqCst) + { return Err(std::io::Error::new( std::io::ErrorKind::ConnectionReset, "lost HEAD response", @@ -1054,6 +1099,14 @@ mod tests { self.entered.notify_one(); self.resume.notified().await; } + if self.fail_remove.load(Ordering::SeqCst) + && self.allow_remove.lock().await.as_ref() != Some(id) + { + return Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "persistent DELETE transport failure", + )); + } let removed = self.inner.remove(space, id, generation).await?; if self.lose_response.swap(false, Ordering::SeqCst) { return Err(std::io::Error::new( @@ -1120,6 +1173,45 @@ mod tests { .await .unwrap(); } + async fn seed_crashed_attempts( + db: &DatabaseConnection, + space: &SpaceId, + value: Hash, + count: usize, + ) { + use sea_orm::ActiveValue::Set; + register(db, &[(space.clone(), value)], "crashed-0").await; + age(db, space, value).await; + let old = time::OffsetDateTime::now_utc() - time::Duration::hours(2); + for start in (1..count).step_by(256) { + kv_orphan_attempt::Entity::insert_many((start..(start + 256).min(count)).map( + |index| kv_orphan_attempt::ActiveModel { + space: Set(SpaceIdWrap(space.clone())), + value: Set(value), + attempt_id: Set(format!("crashed-{index}")), + generation: Set(0), + created_at: Set(old), + retry_after: Set(None), + }, + )) + .exec(db) + .await + .unwrap(); + } + } + + async fn advance_retry_tick(db: &DatabaseConnection, space: &SpaceId, value: Hash) { + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::RetryAfter, + Expr::value(time::OffsetDateTime::now_utc() - Duration::seconds(1)), + ) + .filter(kv_orphan_attempt::Column::Space.eq(SpaceIdWrap(space.clone()))) + .filter(kv_orphan_attempt::Column::Value.eq(value)) + .exec(db) + .await + .unwrap(); + } async fn register( db: &DatabaseConnection, @@ -1617,6 +1709,157 @@ mod tests { assert!(store.contains(&space, &blocked, 0).await.unwrap()); } + #[tokio::test] + async fn repeated_failed_oldest_page_cannot_starve_independent_orphan_across_ticks() { + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let blocked = upload(&inner, &space, b"persistently unreadable orphan").await; + seed_crashed_attempts(&db, &space, blocked, 256).await; + *store.fail_hash.lock().await = Some(blocked); + + let healthy = upload(&inner, &space, b"healthy later orphan").await; + register(&db, &[(space.clone(), healthy)], "healthy").await; + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::hours(1)), + ) + .filter(kv_orphan_attempt::Column::Value.eq(healthy)) + .exec(&db) + .await + .unwrap(); + + // Logical ticks 0, 60, 120 seconds: make the retry deadline due + // before each subsequent production drain without sleeping a minute. + assert!(drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .is_err()); + advance_retry_tick(&db, &space, blocked).await; + assert!(drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .is_err()); + assert!( + !inner.contains(&space, &healthy, 0).await.unwrap(), + "a permanently failing prefix must not hide unrelated bytes" + ); + advance_retry_tick(&db, &space, blocked).await; + assert!(drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .is_err()); + assert!(inner.contains(&space, &blocked, 0).await.unwrap()); + assert_eq!( + inner.total_size(&space).await.unwrap(), + Some(b"persistently unreadable orphan".len() as u64) + ); + } + + #[tokio::test] + async fn four_live_blocked_pages_cannot_starve_independent_orphan_across_ticks() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let blocked = upload(&store, &space, b"live protected orphan").await; + seed_crashed_attempts(&db, &space, blocked, 1024).await; + let live = register(&db, &[(space.clone(), blocked)], "live-writer").await; + + let healthy = upload(&store, &space, b"healthy after four pages").await; + register(&db, &[(space.clone(), healthy)], "healthy").await; + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::hours(1)), + ) + .filter(kv_orphan_attempt::Column::Value.eq(healthy)) + .exec(&db) + .await + .unwrap(); + + assert_eq!( + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &healthy, 0).await.unwrap()); + advance_retry_tick(&db, &space, blocked).await; + assert_eq!( + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .unwrap(), + 1, + "a later healthy hash must progress even across four blocked pages" + ); + assert!(!store.contains(&space, &healthy, 0).await.unwrap()); + advance_retry_tick(&db, &space, blocked).await; + assert_eq!( + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + assert!(store.contains(&space, &blocked, 0).await.unwrap()); + refund_failed(&db, &store, &live, "live-writer") + .await + .unwrap(); + assert_eq!( + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .unwrap(), + 1 + ); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn retried_deletes_cannot_hide_healthy_orphans_across_ticks() { + use sea_orm::ActiveValue::Set; + let db = database().await; + let inner = Arc::new(MemoryStore::default()); + let store = InterruptedDelete::new(inner.clone()); + let space = space(); + let old = time::OffsetDateTime::now_utc() - Duration::hours(2); + kv_orphan_candidate::Entity::insert_many((0..256).map(|index| { + kv_orphan_candidate::ActiveModel { + space: Set(SpaceIdWrap(space.clone())), + value: Set(hash(format!("failed-delete-{index}").as_bytes())), + generation: Set(0), + created_at: Set(old), + deleting_at: Set(Some(old)), + deletion_bytes: Set(Some(1)), + } + })) + .exec(&db) + .await + .unwrap(); + store.fail_remove.store(true, Ordering::SeqCst); + let healthy = upload(&inner, &space, b"healthy behind failed deletes").await; + *store.allow_remove.lock().await = Some(healthy); + register(&db, &[(space.clone(), healthy)], "healthy").await; + age(&db, &space, healthy).await; + assert!(drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .is_err()); + kv_orphan_candidate::Entity::update_many() + .col_expr( + kv_orphan_candidate::Column::DeletingAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::seconds(1)), + ) + .filter(kv_orphan_candidate::Column::DeletingAt.is_not_null()) + .exec(&db) + .await + .unwrap(); + assert!(drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .is_err()); + assert!( + !inner.contains(&space, &healthy, 0).await.unwrap(), + "retrying deletes must leave capacity for independent healthy attempts" + ); + assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + } + #[tokio::test] async fn replayed_registration_requires_own_attempt_without_duplicating_it() { let db = database().await; @@ -2247,44 +2490,6 @@ mod tests { assert!(store.contains(&space, &value, 0).await.unwrap()); } - #[tokio::test] - async fn failed_old_delete_does_not_starve_next_aged_attempt() { - let db = database().await; - let inner = Arc::new(MemoryStore::default()); - let store = InterruptedDelete::new(inner.clone()); - let space = space(); - let old = upload(&inner, &space, b"old failed delete").await; - let next = upload(&inner, &space, b"next candidate").await; - register(&db, &[(space.clone(), old)], "a").await; - register(&db, &[(space.clone(), next)], "b").await; - store.lose_response.store(true, Ordering::SeqCst); - assert!(refund_failed(&db, &store, &[(space.clone(), old, 0)], "a") - .await - .is_err()); - kv_orphan_candidate::Entity::update_many() - .col_expr( - kv_orphan_candidate::Column::DeletingAt, - Expr::value(OffsetDateTime::now_utc() - Duration::hours(2)), - ) - .filter(kv_orphan_candidate::Column::Space.eq(SpaceIdWrap(space.clone()))) - .filter(kv_orphan_candidate::Column::Value.eq(old)) - .exec(&db) - .await - .unwrap(); - age(&db, &space, next).await; - store.lose_response.store(true, Ordering::SeqCst); - assert!(collect_candidates(&db, &store, 1, Duration::hours(1)) - .await - .is_err()); - assert_eq!( - collect_candidates(&db, &store, 1, Duration::hours(1)) - .await - .unwrap(), - 1 - ); - assert!(!inner.contains(&space, &next, 0).await.unwrap()); - } - #[tokio::test] async fn failed_old_metadata_read_does_not_starve_next_attempt() { let db = database().await; diff --git a/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs b/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs index 27ced062..115b9104 100644 --- a/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs +++ b/tinycloud-core/src/migrations/m20261015_020000_kv_orphan_candidates.rs @@ -60,6 +60,18 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_index( + Index::create() + .name("idx_kv_orphan_attempt_retry") + .table(kv_orphan_attempt::Entity) + .col(kv_orphan_attempt::Column::RetryAfter) + .col(kv_orphan_attempt::Column::CreatedAt) + .col(kv_orphan_attempt::Column::Space) + .col(kv_orphan_attempt::Column::Value) + .to_owned(), + ) + .await?; Ok(()) } diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index a5ad9069..41ba54cb 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -49,6 +49,10 @@ pub struct FileSystemStore { #[cfg(test)] before_memory_write: TestPause, #[cfg(test)] + before_pending_refund: TestPause, + #[cfg(test)] + before_reserve_charge: TestPause, + #[cfg(test)] after_memory_contains: TestPause, #[cfg(test)] fail_stage_cleanup: Arc, @@ -69,8 +73,12 @@ impl FileSystemStore { #[cfg(test)] before_memory_write: Arc::new(Mutex::new(None)), #[cfg(test)] + before_pending_refund: Arc::new(Mutex::new(None)), + #[cfg(test)] after_memory_contains: Arc::new(Mutex::new(None)), #[cfg(test)] + before_reserve_charge: Arc::new(Mutex::new(None)), + #[cfg(test)] fail_stage_cleanup: Arc::new(AtomicBool::new(false)), #[cfg(test)] after_stage_unlink: Arc::new(Mutex::new(None)), @@ -96,10 +104,18 @@ impl FileSystemStore { } async fn reserve(&self, space: &SpaceId, hash: Hash, generation: i64, size: u64) { + let key = (space.clone(), hash, generation); let mut pending = self.pending.lock().await; - if let Entry::Vacant(slot) = pending.entry((space.clone(), hash, generation)) { - slot.insert(size); + if let Entry::Vacant(slot) = pending.entry(key) { + #[cfg(test)] + if let Some((entered, release)) = self.before_reserve_charge.lock().await.take() { + entered.notify_one(); + release.notified().await; + } + // Charge before publishing the reservation. Cancellation while + // waiting on SpaceSizes leaves neither a charge nor a claim. self.increment_size(space, size).await; + slot.insert(size); } } @@ -111,14 +127,20 @@ impl FileSystemStore { } async fn refund_pending(&self, space: &SpaceId, hash: Hash, generation: i64) { - let reserved = self - .pending - .lock() - .await - .remove(&(space.clone(), hash, generation)) - .unwrap_or(0); - if reserved != 0 { - self.decrement_size(space, reserved).await; + let key = (space.clone(), hash, generation); + let mut pending = self.pending.lock().await; + if let Some(&reserved) = pending.get(&key) { + #[cfg(test)] + if let Some((entered, release)) = self.before_pending_refund.lock().await.take() { + entered.notify_one(); + release.notified().await; + } + // Retain the refund obligation until the size change completes. + // No await may separate decrement from clearing this claim. + if reserved != 0 { + self.decrement_size(space, reserved).await; + } + pending.remove(&key); } } @@ -1306,6 +1328,99 @@ mod test { assert_eq!(store.remove(&space, &staged_hash, 0).await.unwrap(), None); } + #[tokio::test] + async fn cancelled_failed_upload_retains_refund_until_quota_lock_is_available() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:cancel-refund".parse().unwrap(); + store.create(&space).await.unwrap(); + let bytes = b"failed upload quota"; + let hash = tinycloud_core::hash::hash(bytes); + let target = store.get_path(&space, &hash, 0); + + let writing = Arc::new(Notify::new()); + let resume_write = Arc::new(Notify::new()); + *store.before_memory_write.lock().await = Some((writing.clone(), resume_write.clone())); + let refunding = Arc::new(Notify::new()); + let resume_refund = Arc::new(Notify::new()); + *store.before_pending_refund.lock().await = + Some((refunding.clone(), resume_refund.clone())); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(bytes).await.unwrap(); + let uploading_store = store.clone(); + let uploading_space = space.clone(); + let upload = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &uploading_store, + &uploading_space, + stage, + 0, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), writing.notified()) + .await + .unwrap(); + tokio::fs::create_dir(&target).await.unwrap(); + let quota_lock = store.sizes.lock_updates_for_test().await; + resume_write.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(2), refunding.notified()) + .await + .unwrap(); + resume_refund.notify_one(); + tokio::task::yield_now().await; + upload.abort(); + assert!(upload.await.unwrap_err().is_cancelled()); + drop(quota_lock); + tokio::fs::remove_dir(&target).await.unwrap(); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + + #[tokio::test] + async fn cancelled_reservation_cannot_refund_bytes_never_charged() { + let dir = tempfile::tempdir().unwrap(); + let store = FileSystemConfig::new(dir.path()).open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:cancel-reserve".parse().unwrap(); + store.create(&space).await.unwrap(); + let bytes = b"unacquired charge"; + let hash = tinycloud_core::hash::hash(bytes); + let checked = Arc::new(Notify::new()); + let resume_check = Arc::new(Notify::new()); + *store.after_memory_contains.lock().await = Some((checked.clone(), resume_check.clone())); + let charging = Arc::new(Notify::new()); + let resume_charge = Arc::new(Notify::new()); + *store.before_reserve_charge.lock().await = Some((charging.clone(), resume_charge.clone())); + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(bytes).await.unwrap(); + let uploading_store = store.clone(); + let uploading_space = space.clone(); + let upload = tokio::spawn(async move { + ImmutableWriteStore::::persist( + &uploading_store, + &uploading_space, + stage, + 0, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), checked.notified()) + .await + .unwrap(); + let quota_lock = store.sizes.lock_updates_for_test().await; + resume_check.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(2), charging.notified()) + .await + .unwrap(); + resume_charge.notify_one(); + tokio::task::yield_now().await; + upload.abort(); + assert!(upload.await.unwrap_err().is_cancelled()); + drop(quota_lock); + assert_eq!(store.remove(&space, &hash, 0).await.unwrap(), None); + assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); + } + #[tokio::test] async fn test_file_system_store() { let dir = tempfile::tempdir().unwrap(); diff --git a/tinycloud-node-server/src/storage/size.rs b/tinycloud-node-server/src/storage/size.rs index 9fac85d6..fa1eb60f 100644 --- a/tinycloud-node-server/src/storage/size.rs +++ b/tinycloud-node-server/src/storage/size.rs @@ -26,6 +26,12 @@ impl SpaceSizes { s.sub_assign(size) } } + #[cfg(test)] + pub(crate) async fn lock_updates_for_test( + &self, + ) -> tokio::sync::RwLockWriteGuard<'_, HashMap> { + self.0.write().await + } pub async fn get_size(&self, space: &SpaceId) -> Option { self.0.read().await.get(space).copied() } From 24211501d8403830c7ba8adb0e60f85198bad9e8 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 05:18:40 +0000 Subject: [PATCH 12/17] Keep deferred orphan wakeups in retry queue --- CHANGELOG.md | 2 +- tinycloud-core/src/kv_orphans.rs | 209 ++++++++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78906a8a..c1d30a73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Lost connections during candidate registration retry with the same attempt ID. Neither an upload nor a committed write is replayed on an uncertain outcome. Filesystem-staged S3 bodies now retain their temporary file through send and retry. First uploads keep the bare hash key (generation zero); reuploads after orphan deletion use a new generation-scoped key, so a delayed old delete cannot remove newly committed bytes. Reads obtain the generation in the existing KV lookup. Rolling back to an older binary cannot read generation-above-zero records. Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans. Registration age stays immutable while separate retry eligibility delays blocked claims, so a deferred crash claim cannot masquerade as a live uploader across collectors. Each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. Boot quota reconstruction tracks stage-file charges from the same directory listing as the total; a failed retry and an older crash-leftover stage each refund their own charge after confirmed removal. Known filesystem-stage failures refund their reservation immediately after the unpublished file is removed, and cancellation of a delete cannot interrupt its physical unlink and quota settlement. - Filesystem reservation and refund update pending charges atomically with `SpaceSizes` across cancellation. Each bounded collector page fairly admits never-retried and due-retry attempts through indexed queries, while a full deleting-fence page reserves capacity for expired attempts; persistent storage failures or live same-hash writers cannot starve unrelated orphans across ticks. + Filesystem reservation and refund update pending charges atomically with `SpaceSizes` across cancellation. Each bounded collector page fairly admits never-retried and due-retry attempts through indexed queries, while a full deleting-fence page reserves capacity for expired attempts; persistent storage failures or live same-hash writers cannot starve unrelated orphans across ticks. When a writer departs, deferred siblings become due using DB time without regaining their original first-attempt priority, so repeated departures also cannot starve unrelated orphans. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index ba807063..ed7a8ddd 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -361,7 +361,7 @@ pub async fn finish_candidates( RETURNING space, value ), woken AS ( UPDATE kv_orphan_attempt AS other - SET retry_after = NULL + SET retry_after = statement_timestamp() WHERE other.space = $1 AND other.value = $2 AND other.generation = $4 AND other.attempt_id <> $3 AND other.retry_after IS NOT NULL @@ -426,9 +426,10 @@ pub async fn finish_candidates( Ok(()) } -/// A writer leaving makes older deferred stage claims eligible immediately. -/// Both the successful commit and the failure path call this under the fence; -/// registration age remains immutable throughout. +/// A writer leaving makes older deferred stage claims eligible immediately +/// without restoring their old first-attempt priority. Keep them in the retry +/// queue at the database's current time so an unrelated orphan cannot be +/// starved by repeated departures. Registration age remains immutable. async fn wake_attempts( conn: &C, space: &SpaceIdWrap, @@ -437,7 +438,7 @@ async fn wake_attempts( departing_attempt: &str, ) -> Result<(), DbErr> { attempt::Entity::update_many() - .col_expr(attempt::Column::RetryAfter, Expr::cust("NULL")) + .col_expr(attempt::Column::RetryAfter, Expr::cust("CURRENT_TIMESTAMP")) .filter(attempt::Column::Space.eq(space.clone())) .filter(attempt::Column::Value.eq(value)) .filter(attempt::Column::Generation.eq(generation)) @@ -993,7 +994,10 @@ where mod tests { use super::*; use crate::hash::hash; - use crate::models::{kv_orphan_attempt, kv_orphan_candidate}; + use crate::models::{ + actor, epoch, invocation, kv_orphan_attempt, kv_orphan_candidate, space as space_model, + }; + use crate::relationships::event_order; use crate::storage::{ memory::MemoryStore, HashBuffer, ImmutableReadStore, ImmutableWriteStore, StoreSize, }; @@ -1812,6 +1816,54 @@ mod tests { assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); } + #[tokio::test] + async fn departing_writers_cannot_restore_old_claims_ahead_of_an_unrelated_orphan() { + let db = database().await; + let store = MemoryStore::default(); + let space = space(); + let blocked = upload(&store, &space, b"departing writers").await; + seed_crashed_attempts(&db, &space, blocked, 1024).await; + let mut live = register(&db, &[(space.clone(), blocked)], "live-0").await; + + let healthy = upload(&store, &space, b"unrelated orphan").await; + register(&db, &[(space.clone(), healthy)], "healthy").await; + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::hours(1)), + ) + .filter(kv_orphan_attempt::Column::Value.eq(healthy)) + .exec(&db) + .await + .unwrap(); + + assert_eq!( + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + for index in 1..=2 { + let next = format!("live-{index}"); + let successor = register(&db, &[(space.clone(), blocked)], &next).await; + refund_failed(&db, &store, &live, &format!("live-{}", index - 1)) + .await + .unwrap(); + live = successor; + // A departure wakes old claims while the successor still protects + // their blob. It must not make them outrank a different orphan. + drain_candidates_for_interval(&db, &store, ORPHAN_GRACE) + .await + .unwrap(); + } + assert!( + !store.contains(&space, &healthy, 0).await.unwrap(), + "repeated live writer departures must not starve an unrelated hash" + ); + assert!(store.contains(&space, &blocked, 0).await.unwrap()); + refund_failed(&db, &store, &live, "live-2").await.unwrap(); + } + #[tokio::test] async fn retried_deletes_cannot_hide_healthy_orphans_across_ticks() { use sea_orm::ActiveValue::Set; @@ -2170,6 +2222,114 @@ mod tests { assert_eq!(store.total_size(&space).await.unwrap(), Some(0)); } + #[tokio::test] + async fn postgres_committing_writer_does_not_reprioritize_deferred_orphans() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_committing_writer_does_not_reprioritize_deferred_orphans", + ) + .await + else { + return; + }; + let store = MemoryStore::default(); + let space = space(); + let blocked = upload(&store, &space, b"PG protected backlog").await; + seed_crashed_attempts(&fixture.conn, &space, blocked, 256).await; + register(&fixture.conn, &[(space.clone(), blocked)], "committing").await; + assert_eq!( + collect_candidates(&fixture.conn, &store, 256, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + + let healthy = upload(&store, &space, b"PG unrelated orphan").await; + register(&fixture.conn, &[(space.clone(), healthy)], "healthy").await; + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::hours(1)), + ) + .filter(kv_orphan_attempt::Column::Value.eq(healthy)) + .exec(&fixture.conn) + .await + .unwrap(); + + let actor_id = "did:key:gc-fairness".to_owned(); + actor::Entity::insert(actor::ActiveModel { + id: sea_orm::ActiveValue::Set(actor_id.clone()), + }) + .exec(&fixture.conn) + .await + .unwrap(); + space_model::Entity::insert(space_model::ActiveModel { + id: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), + }) + .exec(&fixture.conn) + .await + .unwrap(); + invocation::Entity::insert(invocation::ActiveModel { + id: sea_orm::ActiveValue::Set(hash(b"invocation")), + invoker: sea_orm::ActiveValue::Set(actor_id), + issued_at: sea_orm::ActiveValue::Set(time::OffsetDateTime::now_utc()), + facts: sea_orm::ActiveValue::Set(None), + serialization: sea_orm::ActiveValue::Set(vec![1]), + }) + .exec(&fixture.conn) + .await + .unwrap(); + epoch::Entity::insert(epoch::ActiveModel { + seq: sea_orm::ActiveValue::Set(1), + id: sea_orm::ActiveValue::Set(hash(b"epoch")), + space: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), + }) + .exec(&fixture.conn) + .await + .unwrap(); + event_order::Entity::insert(event_order::ActiveModel { + seq: sea_orm::ActiveValue::Set(1), + epoch: sea_orm::ActiveValue::Set(hash(b"epoch")), + epoch_seq: sea_orm::ActiveValue::Set(1), + event: sea_orm::ActiveValue::Set(hash(b"invocation")), + space: sea_orm::ActiveValue::Set(SpaceIdWrap(space.clone())), + }) + .exec(&fixture.conn) + .await + .unwrap(); + + let tx = fixture.conn.begin().await.unwrap(); + lock_candidates(&tx, &[(space.clone(), blocked, 0)], "committing") + .await + .unwrap(); + tx.execute(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO kv_write (space, key, invocation, seq, epoch, epoch_seq, value, metadata) + VALUES ($1, 'a', $2, 1, $3, 1, $4, '{}')", + [ + space.to_string().into(), + Vec::::from(hash(b"invocation")).into(), + Vec::::from(hash(b"epoch")).into(), + Vec::::from(blocked).into(), + ], + )) + .await + .unwrap(); + finish_candidates(&tx, &[(space.clone(), blocked, 0)], "committing") + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!( + collect_candidates(&fixture.conn, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 1, + "a writer's commit cannot restore old claims ahead of an unrelated orphan" + ); + assert!(!store.contains(&space, &healthy, 0).await.unwrap()); + assert!(store.contains(&space, &blocked, 0).await.unwrap()); + } + #[tokio::test] async fn postgres_failed_attempt_cannot_erase_another_durable_registration() { let Some(fixture) = crate::test_support::PostgresTestSchema::new( @@ -2453,6 +2613,43 @@ mod tests { ); assert!(!inner.contains(&space, &expired, 0).await.unwrap()); assert_eq!(inner.total_size(&space).await.unwrap(), Some(0)); + + let blocked = upload(&inner, &space, b"mysql-departing-writers").await; + seed_crashed_attempts(&db, &space, blocked, 256).await; + let live = register(&db, &[(space.clone(), blocked)], "mysql-live").await; + assert_eq!( + collect_candidates(&db, &store, 256, ORPHAN_GRACE) + .await + .unwrap(), + 0 + ); + let healthy = upload(&inner, &space, b"mysql-independent-orphan").await; + register(&db, &[(space.clone(), healthy)], "mysql-healthy").await; + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::hours(1)), + ) + .filter(kv_orphan_attempt::Column::Value.eq(healthy)) + .exec(&db) + .await + .unwrap(); + let successor = register(&db, &[(space.clone(), blocked)], "mysql-successor").await; + refund_failed(&db, &store, &live, "mysql-live") + .await + .unwrap(); + assert_eq!( + collect_candidates(&db, &store, 1, ORPHAN_GRACE) + .await + .unwrap(), + 1, + "a departing MySQL writer cannot starve a different hash" + ); + assert!(!inner.contains(&space, &healthy, 0).await.unwrap()); + assert!(inner.contains(&space, &blocked, 0).await.unwrap()); + refund_failed(&db, &store, &successor, "mysql-successor") + .await + .unwrap(); db.close().await.unwrap(); admin .execute(Statement::from_string( From 2800e9efeacf234217e27188672e02fedb976a32 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 11:31:46 +0000 Subject: [PATCH 13/17] fix(TC-735): skip held orphan fences before GC page limit --- CHANGELOG.md | 1 + tinycloud-core/src/db.rs | 4 +- tinycloud-core/src/kv_orphans.rs | 206 ++++++++++++++++-- .../src/storage/file_system.rs | 154 +++++++++++++ 4 files changed, 342 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1d30a73..580b0179 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Physical operations on the same `(space, hash, generation)` now serialize in-process from existence check through upload or deletion and quota settlement; overlapping writers cannot double-charge, and stale S3 HEADs cannot re-create a refunded charge. Filesystem memory staging writes a same-directory temporary file before atomically publishing the final hash key. GC reclaims a crash-leftover stage and its quota charge even if a later same-hash retry committed a reference to the final blob; failed stage cleanup retains the candidate for retry without deleting referenced content. Expired same-hash attempts drain as a group under their fence; a live sibling temporarily defers cleanup without hiding unrelated orphans. Registration age stays immutable while separate retry eligibility delays blocked claims, so a deferred crash claim cannot masquerade as a live uploader across collectors. Each collector tick has a bounded page count. SQLite reserves the database writer before preparatory authorization reads, and populated older PostgreSQL KV backfills decode generation as BIGINT. Boot quota reconstruction tracks stage-file charges from the same directory listing as the total; a failed retry and an older crash-leftover stage each refund their own charge after confirmed removal. Known filesystem-stage failures refund their reservation immediately after the unpublished file is removed, and cancellation of a delete cannot interrupt its physical unlink and quota settlement. Filesystem reservation and refund update pending charges atomically with `SpaceSizes` across cancellation. Each bounded collector page fairly admits never-retried and due-retry attempts through indexed queries, while a full deleting-fence page reserves capacity for expired attempts; persistent storage failures or live same-hash writers cannot starve unrelated orphans across ticks. When a writer departs, deferred siblings become due using DB time without regaining their original first-attempt priority, so repeated departures also cannot starve unrelated orphans. + PostgreSQL selection skips independently held hash fences before its page limit and releases selection locks before storage I/O. A live same-space, same-hash attempt can defer an older crashed stage's physical-byte refund beyond the usual five-minute grace plus 60-second pass. Its upload has a 240-second deadline; the stage becomes due for the next scheduled pass (within 60 seconds) after the last overlapping attempt commits or aborts; a backlog beyond the collector's bounded pages can require further passes. Repeated overlaps extend that delay without deleting committed content or undercounting bytes. The existing epoch-head anti-join still executes inside the per-space sequence-held transaction; its query and indexing are TC-1274, not this change. - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index f633345e..61eff99c 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -54,7 +54,9 @@ use tinycloud_auth::{ /// cap still does; the cap is what bounds HTTP/2 over TLS, whose header list /// may reach 16 MiB. pub const KV_MAX_MUTATIONS_PER_INVOCATION: usize = 4096; -/// All persist calls in one invocation finish before the five-minute orphan GC grace. +/// One invocation's uploads finish within 240 seconds of preflight, before +/// the five-minute orphan age. A live same-hash retry defers an older crash +/// stage until that retry finishes; repeated overlaps can extend its charge. const KV_UPLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(240); const _: () = assert!(KV_UPLOAD_TIMEOUT.as_secs() < crate::kv_orphans::ORPHAN_GRACE.whole_seconds() as u64); diff --git a/tinycloud-core/src/kv_orphans.rs b/tinycloud-core/src/kv_orphans.rs index ed7a8ddd..bc581c28 100644 --- a/tinycloud-core/src/kv_orphans.rs +++ b/tinycloud-core/src/kv_orphans.rs @@ -4,8 +4,8 @@ use std::{borrow::Cow, error::Error as StdError}; use sea_orm::{ sea_query::{Expr, LockBehavior, LockType, OnConflict}, - ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, QueryFilter, - QueryOrder, QuerySelect, Statement, TransactionTrait, + ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, DbErr, EntityTrait, + FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, TransactionTrait, }; use time::{Duration, OffsetDateTime}; use tinycloud_auth::resource::SpaceId; @@ -19,7 +19,17 @@ use crate::storage::{ImmutableDeleteStore, ImmutableReadStore}; use crate::types::SpaceIdWrap; const MAX_BATCH_SIZE: u64 = 256; -/// Expired registrations are collected only after the upload deadline. +/// A failed upload's claim becomes eligible after five minutes of DB time. +/// Ordinary uploads have a 240-second deadline; the server then runs GC every +/// 60 seconds. A genuinely live same-space, same-hash attempt protects an +/// older crash stage and its physical-byte charge until it commits or aborts. +/// The older claim becomes due when the last attempt ends, for the next +/// scheduled pass (within 60 seconds); each upload has at most 240 seconds. +/// A backlog beyond the collector's bounded pages may take further passes. +/// Repeated overlapping attempts can extend this conservative overcharge. +/// A referenced final blob is never deleted or refunded. A single blob key +/// costs at most its physical size; filesystem crash stages are separate +/// physical files whose charges are settled independently. pub const ORPHAN_GRACE: Duration = Duration::minutes(5); const UNAVAILABLE_DELETING: &str = "KV blob is being deleted; retry registration"; @@ -877,25 +887,84 @@ where // Query the two retry states separately so each bounded page can use // its age or retry index. Re-reading an oldest failing page every // minute must not hide an unrelated, never-retried orphan. - let mut never_retried = attempt::Entity::find() - .filter(attempt::Column::CreatedAt.lte(cutoff)) - .filter(attempt::Column::RetryAfter.is_null()) - .order_by_asc(attempt::Column::CreatedAt) - .order_by_asc(attempt::Column::Space) - .order_by_asc(attempt::Column::Value) - .limit(available) - .all(conn) - .await?; - let due_retries = attempt::Entity::find() - .filter(attempt::Column::CreatedAt.lte(cutoff)) - .filter(attempt::Column::RetryAfter.lte(now)) - .order_by_asc(attempt::Column::RetryAfter) - .order_by_asc(attempt::Column::CreatedAt) - .order_by_asc(attempt::Column::Space) - .order_by_asc(attempt::Column::Value) - .limit(available) - .all(conn) - .await?; + // PostgreSQL must exclude held hash fences *before* LIMIT. An ordered + // attempt index scan with a per-row LATERAL fence lock does that while + // keeping the planner off a full hash join/sort of every candidate. + // The retry index also serves NULL retry_after: its leading key is + // constant in that queue. Commit selection before any storage I/O; + // one held hash cannot occupy a page or block an unrelated orphan. + let (mut never_retried, due_retries) = if conn.get_database_backend() == DbBackend::Postgres + { + let tx = conn.begin().await?; + let first = tx + .query_all(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT a.* FROM ( + SELECT * FROM kv_orphan_attempt + WHERE created_at <= $1 AND retry_after IS NULL + ORDER BY retry_after, created_at, space, value + ) AS a + CROSS JOIN LATERAL ( + SELECT 1 FROM kv_orphan_candidate AS fence + WHERE fence.space = a.space AND fence.value = a.value + AND fence.generation = a.generation + FOR UPDATE SKIP LOCKED + ) AS available + LIMIT $2", + [cutoff.into(), (available as i64).into()], + )) + .await?; + let second = tx + .query_all(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT a.* FROM ( + SELECT * FROM kv_orphan_attempt + WHERE created_at <= $1 AND retry_after <= $2 + ORDER BY retry_after, created_at, space, value + ) AS a + CROSS JOIN LATERAL ( + SELECT 1 FROM kv_orphan_candidate AS fence + WHERE fence.space = a.space AND fence.value = a.value + AND fence.generation = a.generation + FOR UPDATE SKIP LOCKED + ) AS available + LIMIT $3", + [cutoff.into(), now.into(), (available as i64).into()], + )) + .await?; + tx.commit().await?; + ( + first + .iter() + .map(|row| attempt::Model::from_query_result(row, "")) + .collect::, _>>()?, + second + .iter() + .map(|row| attempt::Model::from_query_result(row, "")) + .collect::, _>>()?, + ) + } else { + let never_retried = attempt::Entity::find() + .filter(attempt::Column::CreatedAt.lte(cutoff)) + .filter(attempt::Column::RetryAfter.is_null()) + .order_by_asc(attempt::Column::CreatedAt) + .order_by_asc(attempt::Column::Space) + .order_by_asc(attempt::Column::Value) + .limit(available) + .all(conn) + .await?; + let due_retries = attempt::Entity::find() + .filter(attempt::Column::CreatedAt.lte(cutoff)) + .filter(attempt::Column::RetryAfter.lte(now)) + .order_by_asc(attempt::Column::RetryAfter) + .order_by_asc(attempt::Column::CreatedAt) + .order_by_asc(attempt::Column::Space) + .order_by_asc(attempt::Column::Value) + .limit(available) + .all(conn) + .await?; + (never_retried, due_retries) + }; never_retried.extend(due_retries); never_retried.sort_unstable_by(|left, right| { left.retry_after @@ -2188,6 +2257,99 @@ mod tests { ); } + #[tokio::test] + async fn postgres_held_old_fence_does_not_starve_an_independent_orphan_across_ticks() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_held_old_fence_does_not_starve_an_independent_orphan_across_ticks", + ) + .await + else { + return; + }; + let store = MemoryStore::default(); + let space = space(); + let blocked = upload(&store, &space, b"held oldest fence").await; + seed_crashed_attempts(&fixture.conn, &space, blocked, 256).await; + let healthy = upload(&store, &space, b"independent expired blob").await; + register(&fixture.conn, &[(space.clone(), healthy)], "healthy").await; + kv_orphan_attempt::Entity::update_many() + .col_expr( + kv_orphan_attempt::Column::CreatedAt, + Expr::value(time::OffsetDateTime::now_utc() - Duration::hours(1)), + ) + .filter(kv_orphan_attempt::Column::Value.eq(healthy)) + .exec(&fixture.conn) + .await + .unwrap(); + + let schema: String = fixture + .conn + .query_one(Statement::from_string( + DbBackend::Postgres, + "SELECT current_schema() AS schema_name", + )) + .await + .unwrap() + .unwrap() + .try_get("", "schema_name") + .unwrap(); + let mut options = ConnectOptions::new( + crate::test_support::postgres_test_url( + "postgres_held_old_fence_does_not_starve_an_independent_orphan_across_ticks", + ) + .unwrap(), + ); + options.set_schema_search_path(schema); + let collector = Database::connect(options).await.unwrap(); + let held = fixture.conn.begin().await.unwrap(); + lock_candidates(&held, &[(space.clone(), blocked, 0)], "crashed-0") + .await + .unwrap(); + + // Logical ticks 0/60/120 advance the age cutoff without wall-clock + // sleeps. The other pool holds X throughout; H must make progress. + let mut ticks = Vec::with_capacity(3); + for tick in 0..3 { + let min_age = ORPHAN_GRACE - Duration::seconds(tick * 60); + let snapshot = match tokio::time::timeout( + std::time::Duration::from_secs(20), + drain_candidates_for_interval(&collector, &store, min_age), + ) + .await + { + Ok(Ok(removed)) => ( + removed, + store.contains(&space, &healthy, 0).await.unwrap(), + store.contains(&space, &blocked, 0).await.unwrap(), + ), + _ => (usize::MAX, true, true), + }; + ticks.push(snapshot); + if snapshot.0 == usize::MAX { + break; + } + } + // Always release the independent transaction and pool before checking + // the assertions; otherwise a RED test can deadlock schema teardown. + held.rollback().await.unwrap(); + let after_release = tokio::time::timeout( + std::time::Duration::from_secs(20), + drain_candidates_for_interval(&collector, &store, ORPHAN_GRACE), + ) + .await; + let blocked_after_release = store.contains(&space, &blocked, 0).await.unwrap(); + let final_size = store.total_size(&space).await.unwrap(); + collector.close().await.unwrap(); + assert_eq!( + ticks, + vec![(1, false, true), (0, false, true), (0, false, true)], + "the unrelated H must retire while X's fence is still held" + ); + assert_eq!(after_release.unwrap().unwrap(), 1); + assert!(!blocked_after_release); + assert_eq!(final_size, Some(0)); + } + #[tokio::test] async fn postgres_expired_same_hash_claims_drain_under_one_fence() { let Some(fixture) = crate::test_support::PostgresTestSchema::new( diff --git a/tinycloud-node-server/src/storage/file_system.rs b/tinycloud-node-server/src/storage/file_system.rs index 41ba54cb..f86a5692 100644 --- a/tinycloud-node-server/src/storage/file_system.rs +++ b/tinycloud-node-server/src/storage/file_system.rs @@ -1289,6 +1289,160 @@ mod test { ); } + #[tokio::test] + async fn expired_crash_stage_is_refunded_after_overlapping_retry_commits_or_aborts() { + use tinycloud_core::sea_orm::{ + ConnectionTrait, Database, DbBackend, EntityTrait, PaginatorTrait, Statement, + TransactionTrait, + }; + use tinycloud_core::sea_orm_migration::MigratorTrait; + use tinycloud_core::{ + hash::hash, + kv_orphans, + migrations::Migrator, + models::{kv_orphan_attempt, kv_orphan_candidate, kv_write}, + }; + + for commits in [true, false] { + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + db.execute_unprepared("PRAGMA foreign_keys = OFF") + .await + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let cfg = FileSystemConfig::new(dir.path()); + let store = cfg.open().await.unwrap(); + let space: SpaceId = "tinycloud:key:test:overlap-bound".parse().unwrap(); + store.create(&space).await.unwrap(); + let data = b"complete retry after crashed stage"; + let value = hash(data); + let blobs = [(space.clone(), value)]; + let tx = db.begin().await.unwrap(); + kv_orphans::register_candidates(&tx, &blobs, "crashed-A") + .await + .unwrap(); + tx.commit().await.unwrap(); + let target = store.get_path(&space, &value, 0); + let mut stage = tempfile::Builder::new() + .prefix(&FileSystemStore::stage_prefix(&target)) + .tempfile_in(target.parent().unwrap()) + .unwrap(); + std::io::Write::write_all(&mut stage, b"partial").unwrap(); + let stage_path = stage.path().to_path_buf(); + stage.into_temp_path().keep().unwrap(); + drop(store); + let store = cfg.open().await.unwrap(); + assert_eq!(store.total_size(&space).await.unwrap(), Some(7)); + db.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "UPDATE kv_orphan_attempt SET created_at = ? WHERE attempt_id = ?", + [ + (time::OffsetDateTime::now_utc() - time::Duration::minutes(6)).into(), + "crashed-A".into(), + ], + )) + .await + .unwrap(); + + let tx = db.begin().await.unwrap(); + let retry = kv_orphans::register_candidates(&tx, &blobs, "live-B") + .await + .unwrap(); + tx.commit().await.unwrap(); + let mut upload = HashBuffer::new(Vec::new()); + upload.write_all(data).await.unwrap(); + assert_eq!( + ImmutableWriteStore::::persist( + &store, &space, upload, retry[0].2 + ) + .await + .unwrap(), + value + ); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(7 + data.len() as u64) + ); + assert_eq!( + kv_orphans::collect_candidates(&db, &store, 256, kv_orphans::ORPHAN_GRACE) + .await + .unwrap(), + 0, + "A's expired stage must remain charged while B is live" + ); + assert!(stage_path.exists()); + + if commits { + let tx = db.begin().await.unwrap(); + kv_orphans::lock_candidates(&tx, &retry, "live-B") + .await + .unwrap(); + tx.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "INSERT INTO kv_write (space,key,invocation,seq,epoch,epoch_seq,value,metadata,generation) + VALUES (?,'retry',?,1,?,1,?,'{}',?)", + [ + space.to_string().into(), + Vec::::from(hash(b"invocation")).into(), + Vec::::from(hash(b"epoch")).into(), + Vec::::from(value).into(), + retry[0].2.into(), + ], + )) + .await + .unwrap(); + kv_orphans::finish_candidates(&tx, &retry, "live-B") + .await + .unwrap(); + tx.commit().await.unwrap(); + } else { + kv_orphans::refund_failed(&db, &store, &retry, "live-B") + .await + .unwrap(); + } + + assert_eq!( + kv_orphans::collect_candidates(&db, &store, 256, kv_orphans::ORPHAN_GRACE) + .await + .unwrap(), + usize::from(!commits), + "A's crash charge must settle on the next pass after B leaves" + ); + assert!(!stage_path.exists()); + assert_eq!( + store.read_to_vec(&space, &value, 0).await.unwrap(), + commits.then(|| data.to_vec()), + "a committed retry must remain readable; an aborted one must be absent" + ); + assert_eq!( + kv_write::Entity::find().count(&db).await.unwrap(), + u64::from(commits) + ); + assert_eq!( + store.total_size(&space).await.unwrap(), + Some(if commits { data.len() as u64 } else { 0 }) + ); + assert_eq!( + kv_orphans::collect_candidates(&db, &store, 256, kv_orphans::ORPHAN_GRACE) + .await + .unwrap(), + 0, + "settled crash attempts cannot refund twice" + ); + assert_eq!( + kv_orphan_candidate::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); + assert_eq!( + kv_orphan_attempt::Entity::find().count(&db).await.unwrap(), + 0 + ); + } + } + #[tokio::test] async fn failed_file_creation_refunds_reserved_quota() { let dir = tempfile::tempdir().unwrap(); From 580abe562cf20c06b32a5124e76f0c0502ee30d8 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 11:52:18 +0000 Subject: [PATCH 14/17] Resolve CID-alias revocations inside the batched auth snapshot TC-1071 revokes every stored encoding sharing a canonical identity. Add an alias-joined revocation branch to the single snapshot statement, mirroring revocation::effective_rows, and drop the merge's extra query. Regression: a revocation under another encoding is refused on SQLite, PostgreSQL and MySQL. A seeded property test compares production batched decisions with the sequential effective_rows snapshot over random DAGs, alias groups, revoked and expired delegations. --- CHANGELOG.md | 2 +- tinycloud-core/src/auth_graph.rs | 270 +++++++++++++++++++++--- tinycloud-core/src/models/invocation.rs | 13 ++ 3 files changed, 255 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74f71427..a2ccf085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- Load each invocation's authorization graph in one SQL round trip on PostgreSQL, SQLite and MySQL rather than four. Bound metadata lookups to 65 nodes for indexed access on large tables while the unchanged 64-node authorization limit still denies oversized chains. Sort abilities by `(resource, ability)` on every backend; authorization decisions are unchanged, but the diagnostic reason when multiple caveats all reject may normalize. No schema change (TC-735). +- Load each invocation's authorization graph in one SQL round trip on PostgreSQL, SQLite and MySQL rather than four. Revocations in that statement resolve through the persisted CID-alias index exactly like `revocation::effective_rows`, so a delegation revoked under another accepted encoding is still refused (TC-1071). Bound metadata lookups to 65 nodes for indexed access on large tables while the unchanged 64-node authorization limit still denies oversized chains. Sort abilities by `(resource, ability)` on every backend; authorization decisions are unchanged, but the diagnostic reason when multiple caveats all reject may normalize. No schema change (TC-735). - Request events include validated/echoed UUID correlation, typed replacement flags, node-keyed target account/space pseudonyms and per-request database acquisition/execution timings and attempt counts. Driver/actor instrumentation includes failures and request-awaited audit batches; forwarding requires explicit first-party origin classification. A shared core `Pseudonymizer` preserves existing account/space hashes, adds the service/space-scoped database hash primitive for F6, and is derived once at startup for injection into consumers. See `docs/request-telemetry.md` for the wire contract, key custody, restart joinability and coverage/exclusions (TC-959 N2). - CACAO/SIWE delegation ingestion normalizes ECDSA recovery IDs 0/1 to 27/28 before hashing and persisting the canonical DAG-CBOR bytes (TC-1071). Equivalent low-s signatures now share one delegation CID, including the CID used for revocation lookup. High-s signatures are rejected during decoding, matching the existing EIP-191 verifier and EIP-2 low-s convention; UCAN/JWS bytes remain unchanged. Existing stored delegation IDs are not rewritten. - Synchronize the standalone M1, W5 and load-signer Cargo locks with CACAO's `k256` dependency so `--locked` dependency resolution succeeds (TC-1071). Keep the load signer compatible with the SDK's default invocation lifetime. diff --git a/tinycloud-core/src/auth_graph.rs b/tinycloud-core/src/auth_graph.rs index da87c6b2..44c7e050 100644 --- a/tinycloud-core/src/auth_graph.rs +++ b/tinycloud-core/src/auth_graph.rs @@ -469,33 +469,51 @@ fn snapshot_query(roots: &[Hash]) -> WithQuery { ) .and_where(within_budget()); query.union(UnionType::All, abilities_query); - let mut revocations = snapshot_branch( - 3, - [ - node(lookup_name.clone()), - null(), - null(), - null(), - null(), - null(), - null(), - null(), - null(), - null(), - null(), - null(), - ], - ); + let revocation_fields = || { + let mut fields: [SimpleExpr; 12] = std::array::from_fn(|_| null()); + fields[0] = node(lookup_name.clone()); + fields + }; + let mut revocations = snapshot_branch(3, revocation_fields()); revocations .from(revocation::Entity) .join( JoinType::InnerJoin, - lookup_name, + lookup_name.clone(), Expr::col((revocation::Entity, revocation::Column::Revoked)) - .equals((Alias::new("lookup_nodes"), id)), + .equals((lookup_name.clone(), id.clone())), ) .and_where(within_budget()); query.union(UnionType::All, revocations); + // TC-1071: revoking any stored encoding revokes every encoding with the + // same canonical identity. This mirrors `revocation::effective_rows`' + // alias branch inside the same statement; signed targets stay unchanged. + let source = Alias::new("revocation_source_alias"); + let target = Alias::new("revocation_target_alias"); + let mut aliased_revocations = snapshot_branch(3, revocation_fields()); + aliased_revocations + .from(lookup_name.clone()) + .join_as( + JoinType::InnerJoin, + delegation_cid_alias::Entity, + source.clone(), + Expr::col((source.clone(), delegation_cid_alias::Column::Id)).equals((lookup_name, id)), + ) + .join_as( + JoinType::InnerJoin, + delegation_cid_alias::Entity, + target.clone(), + Expr::col((source, delegation_cid_alias::Column::Canonical)) + .equals((target.clone(), delegation_cid_alias::Column::Canonical)), + ) + .join( + JoinType::InnerJoin, + revocation::Entity, + Expr::col((target, delegation_cid_alias::Column::Id)) + .equals((revocation::Entity, revocation::Column::Revoked)), + ) + .and_where(within_budget()); + query.union(UnionType::All, aliased_revocations); query.with( WithClause::new() .recursive(true) @@ -620,11 +638,6 @@ impl AuthGraphSnapshot { } } - let revoked = revocation::effective_rows(db, &nodes) - .await? - .into_iter() - .map(|(id, _)| id) - .collect(); Ok(Self { parents, delegations, @@ -792,12 +805,12 @@ mod tests { // Compare the old loader's rows under the new explicit order, rather // than accidentally pinning one backend's unspecified row order. sort_abilities(&mut ability_rows); - let revoked = revocation::Entity::find() - .filter(revocation::Column::Revoked.is_in(nodes.iter().copied())) - .all(db) + // Main's sequential TC-1071 semantics: revocations under any + // persisted encoding of a delegation apply to every encoding. + let revoked = revocation::effective_rows(db, &nodes) .await? .into_iter() - .map(|row| row.revoked) + .map(|(id, _)| id) .collect(); Ok(AuthGraphSnapshot { parents, @@ -1020,6 +1033,198 @@ mod tests { assert_eq!(snapshot, load_sequential(db, &[ids[0]]).await.unwrap()); } + async fn insert_alias(db: &DatabaseConnection, id: Hash, canonical: Hash) { + delegation_cid_alias::ActiveModel { + id: Set(id), + canonical: Set(canonical), + encrypted: Set(false), + } + .insert(db) + .await + .unwrap(); + } + + async fn insert_revocation(db: &DatabaseConnection, id: Hash, target: Hash) { + revocation::ActiveModel { + id: Set(id), + revoker: Set("did:key:actor".to_string()), + revoked: Set(target), + serialization: Set(id.as_ref().to_vec()), + revoked_at: Set(Some(OffsetDateTime::now_utc())), + } + .insert(db) + .await + .unwrap(); + } + + /// TC-1071: a revocation names one stored encoding; every other stored + /// encoding with the same canonical identity is revoked too. The batched + /// snapshot must refuse exactly where `revocation::effective_rows` does. + async fn assert_alias_revocation_refused(db: &DatabaseConnection) { + let ids = insert_chain(db, "alias-revoked", 2).await; + let other = hash(b"alias-revoked-other-encoding"); + insert_delegation(db, other).await; + let canonical = hash(b"alias-revoked-canonical"); + insert_alias(db, ids[1], canonical).await; + insert_alias(db, other, canonical).await; + insert_revocation(db, hash(b"alias-revoked-revocation"), other).await; + + assert!(revocation::is_revoked(db, &ids[1]).await.unwrap()); + let snapshot = AuthGraphSnapshot::load(db, &[ids[0]]).await.unwrap(); + assert!( + snapshot.is_revoked(&ids[1]), + "batched snapshot missed a revocation under another encoding" + ); + assert_eq!( + snapshot.first_revoked_ancestor(&ids[0]), + Some(ids[1].to_cid(0x55).to_string()) + ); + assert_eq!(snapshot, load_sequential(db, &[ids[0]]).await.unwrap()); + let cited = AuthGraphSnapshot::load(db, &[ids[1]]).await.unwrap(); + assert!(cited.is_revoked(&ids[1])); + assert!(!cited.is_revoked(&ids[2])); + } + + /// Seeded property check: over random DAGs with mixed alias groups, + /// revocations of cited and uncited encodings, and expired delegations, + /// the production batched decision equals the decision computed from the + /// sequential `effective_rows` snapshot and the per-node DB lookups. + async fn assert_batched_matches_sequential_decisions( + db: &DatabaseConnection, + backend: &str, + cases: u64, + ) { + use rand::{rngs::StdRng, Rng, SeedableRng}; + let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap(); + let capability = crate::util::Capability { + resource: "tinycloud:did:key:actor:files/kv/doc".parse().unwrap(), + ability: "tinycloud.kv/get".to_string().try_into().unwrap(), + caveats: Caveats::default(), + }; + let (mut alias_refusals, mut accepted, mut refused) = (0, 0, 0); + for case in 0..cases { + let seed = 0x7c73_5000 + case; + let mut rng = StdRng::seed_from_u64(seed); + let tag = format!("prop-{backend}-{seed}"); + let node = |kind: &str, index: usize| hash(format!("{tag}-{kind}-{index}").as_bytes()); + let count = rng.gen_range(1..=6); + let ids: Vec = (0..count).map(|index| node("delegation", index)).collect(); + // Other stored encodings of a canonical delegation; never cited. + let encodings: Vec = (0..rng.gen_range(0..=3)) + .map(|index| node("encoding", index)) + .collect(); + let canonicals = [node("canonical", 0), node("canonical", 1)]; + for id in ids.iter().chain(&encodings) { + let expiry = match rng.gen_range(0..4) { + 0 => Some(now - time::Duration::minutes(5)), + 1 => Some(now + time::Duration::hours(1)), + _ => None, + }; + delegation::ActiveModel { + id: Set(*id), + delegator: Set("did:key:actor".to_string()), + delegatee: Set("did:key:alias".to_string()), + expiry: Set(expiry), + issued_at: Set(None), + not_before: Set(None), + facts: Set(None), + serialization: Set(id.as_ref().to_vec()), + } + .insert(db) + .await + .unwrap(); + abilities::ActiveModel { + delegation: Set(*id), + resource: Set("tinycloud:did:key:actor:files/kv".parse().unwrap()), + ability: Set("tinycloud.kv/get".to_string().try_into().unwrap()), + caveats: Set(Caveats::default()), + } + .insert(db) + .await + .unwrap(); + if rng.gen_bool(0.6) { + insert_alias(db, *id, canonicals[rng.gen_range(0..2)]).await; + } + } + for child in 1..count { + let mut parents = HashSet::new(); + for _ in 0..rng.gen_range(0..=2) { + parents.insert(rng.gen_range(0..child)); + } + for parent in parents { + insert_edge(db, ids[child], ids[parent]).await; + } + } + let mut targets = HashSet::new(); + for id in ids.iter().chain(&encodings) { + if rng.gen_bool(0.2) { + targets.insert(*id); + } + } + for (index, target) in targets.iter().enumerate() { + insert_revocation(db, node("revocation", index), *target).await; + } + + let mut root_sets: Vec> = ids.iter().map(|id| vec![*id]).collect(); + root_sets.push(ids.clone()); + for roots in root_sets { + let context = format!("{backend} seed {seed} roots {roots:?}"); + let batched = AuthGraphSnapshot::load(db, &roots).await.unwrap(); + let sequential = load_sequential(db, &roots).await.unwrap(); + assert_eq!(batched, sequential, "{context}"); + for root in &roots { + let revoked = revocation::is_revoked(db, root).await.unwrap(); + assert_eq!(batched.is_revoked(root), revoked, "{context}"); + assert_eq!( + batched.first_revoked_ancestor(root).is_some(), + revocation::first_revoked_ancestor(db, root) + .await + .unwrap() + .is_some(), + "{context}" + ); + if revoked && !targets.contains(root) { + alias_refusals += 1; + } + } + let parents: Vec<_> = roots.iter().map(|id| id.to_cid(0x55)).collect(); + let production = crate::models::invocation::authorize_holder( + db, + "did:key:alias", + std::slice::from_ref(&capability), + &parents, + now, + ) + .await; + let reference = crate::models::invocation::authorize_holder_with_graph( + db, + "did:key:alias", + std::slice::from_ref(&capability), + &parents, + now, + &sequential, + ) + .await; + assert_eq!( + format!("{production:?}"), + format!("{reference:?}"), + "{context}" + ); + if production.is_ok() { + accepted += 1; + } else { + refused += 1; + } + } + } + // The generator must exercise every decision class it claims to. + assert!(alias_refusals > 0, "{backend}: no alias-only revocation"); + assert!( + accepted > 0 && refused > 0, + "{backend}: one-sided decisions" + ); + } + #[tokio::test] async fn sqlite_snapshot_matches_sequential_for_chains_windows_caveats_revocations_aliases() { let db = Database::connect("sqlite::memory:").await.unwrap(); @@ -1027,6 +1232,8 @@ mod tests { assert_fixture_equivalence(&db).await; assert_canonical_caveat_reason(&db).await; assert_closure_limit(&db).await; + assert_alias_revocation_refused(&db).await; + assert_batched_matches_sequential_decisions(&db, "sqlite", 200).await; } #[tokio::test] @@ -1041,6 +1248,8 @@ mod tests { assert_fixture_equivalence(&fixture.conn).await; assert_canonical_caveat_reason(&fixture.conn).await; assert_closure_limit(&fixture.conn).await; + assert_alias_revocation_refused(&fixture.conn).await; + assert_batched_matches_sequential_decisions(&fixture.conn, "postgres", 60).await; } struct MySqlGraphFixture { @@ -1102,7 +1311,7 @@ mod tests { }; let db = Database::connect(format!("{prefix}/{name}")).await.unwrap(); // The full repository migrator predates MySQL's indexed-column length - // limits. Keep this fixture scoped to the four auth-graph model tables + // limits. Keep this fixture scoped to the auth-graph model tables // rather than changing unrelated migrations for a query equivalence test. for ddl in [ "CREATE TABLE actor (id VARCHAR(191) PRIMARY KEY)", @@ -1110,6 +1319,7 @@ mod tests { "CREATE TABLE parent_delegation (child VARBINARY(34) NOT NULL, parent VARBINARY(34) NOT NULL, PRIMARY KEY (parent, child))", "CREATE TABLE ability (resource VARCHAR(255) NOT NULL, ability VARCHAR(255) NOT NULL, delegation VARBINARY(34) NOT NULL, caveats JSON NOT NULL, PRIMARY KEY (resource, ability, delegation))", "CREATE TABLE revocation (id VARBINARY(34) PRIMARY KEY, revoker VARCHAR(191) NOT NULL, revoked VARBINARY(34) NOT NULL, serialization BLOB NOT NULL, revoked_at DATETIME NULL)", + "CREATE TABLE delegation_cid_alias (id VARBINARY(64) PRIMARY KEY, canonical VARBINARY(64) NOT NULL, encrypted BOOLEAN NOT NULL, INDEX idx_delegation_cid_alias_canonical (canonical, id))", ] { db.execute(Statement::from_string(DbBackend::MySql, ddl.to_string())) .await @@ -1118,6 +1328,8 @@ mod tests { assert_fixture_equivalence(&db).await; assert_canonical_caveat_reason(&db).await; assert_closure_limit(&db).await; + assert_alias_revocation_refused(&db).await; + assert_batched_matches_sequential_decisions(&db, "mysql", 60).await; db.close().await.unwrap(); admin.close().await.unwrap(); } diff --git a/tinycloud-core/src/models/invocation.rs b/tinycloud-core/src/models/invocation.rs index c112506b..3874d34b 100644 --- a/tinycloud-core/src/models/invocation.rs +++ b/tinycloud-core/src/models/invocation.rs @@ -299,6 +299,19 @@ pub async fn authorize_holder( validate_capabilities(db, capabilities, holder, parents, Some(now), None).await } +/// Test oracle: `authorize_holder`'s checks against a caller-built snapshot. +#[cfg(test)] +pub(crate) async fn authorize_holder_with_graph( + db: &C, + holder: &str, + capabilities: &[util::Capability], + parents: &[tinycloud_auth::ipld_core::cid::Cid], + now: OffsetDateTime, + graph: &crate::auth_graph::AuthGraphSnapshot, +) -> Result<(), Error> { + validate_capabilities(db, capabilities, holder, parents, Some(now), Some(graph)).await +} + async fn validate_capabilities( db: &C, capabilities: &[util::Capability], From 94414d519bd92c0532ad3ca5949da02528ee2d1e Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 13:21:08 +0000 Subject: [PATCH 15/17] Roll the cursor-index lock test back from its own migration TC-735's later kv_orphan_candidates migration made Migrator::down(1) undo it instead of the cursor index. Identical to #341's fix. --- .../migrations/m20261015_000000_delegation_cursor_index.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs b/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs index 9083d249..07fdc662 100644 --- a/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs +++ b/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs @@ -107,7 +107,12 @@ mod tests { }; let db = &fixture.conn; // Back to the state a node upgrading to this migration boots with. - crate::migrations::Migrator::down(db, Some(1)) + let migrations = crate::migrations::Migrator::migrations(); + let this = migrations + .iter() + .position(|migration| migration.name() == Migration.name()) + .unwrap(); + crate::migrations::Migrator::down(db, Some((migrations.len() - this) as u32)) .await .unwrap(); assert!(!recorded(db).await); From 9050d259972f57d180ef02f8885c06e46b98817e Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 14:21:42 +0000 Subject: [PATCH 16/17] Format the generation-0 S3 block store test --- tinycloud-node-server/tests/s3_block_store.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tinycloud-node-server/tests/s3_block_store.rs b/tinycloud-node-server/tests/s3_block_store.rs index 718da4f1..7243b9b0 100644 --- a/tinycloud-node-server/tests/s3_block_store.rs +++ b/tinycloud-node-server/tests/s3_block_store.rs @@ -72,7 +72,10 @@ async fn remove_deletes_the_block_and_releases_its_size() -> Result<()> { !store.contains(&space, &removed, 0).await?, "object still exists" ); - ensure!(store.contains(&space, &kept, 0).await?, "other block removed"); + ensure!( + store.contains(&space, &kept, 0).await?, + "other block removed" + ); ensure!( store.total_size(&space).await? == Some(17), "size drops by the removed block's length" From 3ca9ceda693ef48272cf1d62f4595bf16839acf1 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 15:01:38 +0000 Subject: [PATCH 17/17] Keep KV upload and post-commit reads out of the commit future frame CI aborted policy_owner_may_be_a_session_the_space_owner_delegated_to on the 2 MiB test stack. gdb at the overflow: the deepest path is invoke -> boxed commit block -> transact -> process_admitted -> save. transact's debug frame (~188 KB) and the test's own frame (~936 KB) are the same as main. This branch adds the boxed commit block's frame (106,608 bytes) under the outer invoke frame. In debug builds every awaited temporary in that block has its own stack slot. Move the upload loop and the post-transact reads into their own methods, so their temporaries are on the stack only while they run. The block's frame is now 78,208 bytes. The test's minimum passing stack falls from 2,056 KiB to 2,028 KiB (main: 2,032 KiB). The test stack is unchanged. --- tinycloud-core/src/db.rs | 318 +++++++++++++++++++++++---------------- 1 file changed, 187 insertions(+), 131 deletions(-) diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index 25bdc8ed..d70fc94e 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -2415,33 +2415,13 @@ where None }; if !blobs.is_empty() { - if tokio::time::Instant::now() >= upload_deadline { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "KV upload deadline elapsed before persist", - ) - .into()); - } - tokio::time::timeout_at(upload_deadline, async { - for (key, stage) in stages.drain() { - let expected = write_hashes[&key]; - let (space, _) = key; - let index = blobs - .binary_search_by(|(s, v, _)| (s, v).cmp(&(&space, &expected))) - .expect("registered generation for every staged write"); - let hash = self - .storage - .persist(&space, stage, blobs[index].2) - .await - .map_err(TxStoreError::StoreWrite)?; - debug_assert_eq!(hash, expected); - } - Ok::<(), TxStoreError>(()) - }) - .await - .map_err(|_| { - std::io::Error::new(std::io::ErrorKind::TimedOut, "KV upload timed out") - })??; + self.persist_registered_blobs::( + std::mem::take(&mut stages), + &write_hashes, + &blobs, + upload_deadline, + ) + .await?; } // PG sequence guards are taken after upload. SQLite takes them // before its writer mutex and BEGIN. The protected SQLite writer @@ -2562,110 +2542,17 @@ where ) .await?; - let mut results = Vec::new(); - // perform and record side effects - for cap in caps.iter().filter_map(|c| { - c.resource.tinycloud_resource().and_then(|r| { - Some(( - r.space(), - r.service().as_str(), - // TC-119: resolve deprecated aliases to canonical (see the - // staging loop above) — identity for canonical URNs. - crate::policy_capability::resolve_alias(c.ability.as_ref().as_ref()), - r.path()?, - )) - }) - }) { - match cap { - (space, "kv", "tinycloud.kv/get", path) => { - let data = - get_kv(&tx, &self.storage, space, path) - .await - .map_err(|e| match e { - EitherError::A(e) => TxStoreError::Tx(e.into()), - EitherError::B(e) => TxStoreError::StoreRead(e), - })?; - if let (Some(limit), Some((_, _, content))) = - (options.max_response_bytes, data.as_ref()) - { - if content.len() > limit { - return Err(TxStoreError::KvResponseTooLarge { - size: content.len(), - limit, - }); - } - } - results.push(InvocationOutcome::KvRead(data)); - } - (space, "kv", "tinycloud.kv/list", path) => { - let (list, truncated) = list_bounded_after( - &tx, - space, - path, - options.list_limit, - options.list_cursor.as_ref(), - ) - .await?; - results.push(InvocationOutcome::KvList(list, truncated, None)) - } - (space, "kv", "tinycloud.kv/del", path) => { - // KV deletion is logical. Blobs are content-addressed and may be - // shared by live sibling keys or retained version history. - results.push(InvocationOutcome::KvDelete( - deleted_hashes.get(&(space.clone(), path.clone())).copied(), - )) - } - (space, "kv", "tinycloud.kv/put", path) => { - let hash = write_hashes - .get(&(space.clone(), path.clone())) - .copied() - .expect("staged KV writes have a content hash"); - results.push(InvocationOutcome::KvWrite(hash)); - } - (space, "kv", "tinycloud.kv/metadata", path) => results.push( - InvocationOutcome::KvMetadata(metadata_with_hash(&tx, space, path).await?), - ), - (space, "capabilities", "tinycloud.capabilities/read", path) - if path.as_str() == "all" => - { - match &caps_read_params { - None => { - // Backward compatible: no params means return all valid delegations - results.push(InvocationOutcome::OpenSessions( - get_valid_delegations(&tx, space, self.encryption.as_ref()) - .await?, - )) - } - Some(CapabilitiesReadParams::List { filters }) => { - // List with optional filters - results.push(InvocationOutcome::OpenSessions( - get_filtered_delegations( - &tx, - space, - &invoker, - filters.as_ref(), - self.encryption.as_ref(), - ) - .await?, - )) - } - Some(CapabilitiesReadParams::Chain { delegation_cid }) => { - // Get the delegation chain for a specific delegation - results.push(InvocationOutcome::DelegationChain( - get_delegation_chain( - &tx, - space, - delegation_cid, - self.encryption.as_ref(), - ) - .await?, - )) - } - } - } - _ => {} - }; - } + let results = self + .invocation_side_effects::( + &tx, + &caps, + &invoker, + &options, + &write_hashes, + &deleted_hashes, + caps_read_params.as_ref(), + ) + .await?; if !blobs.is_empty() { crate::kv_orphans::finish_candidates(&tx, &blobs, &attempt_id).await?; } @@ -2696,6 +2583,175 @@ where result } + /// Upload staged KV blobs under their registered generations before any + /// sequence lock or KV transaction. + /// + /// A separate function, not inline in the commit future: debug builds give + /// every awaited temporary its own stack slot, and the commit future runs + /// the deepest request path (TC-735 #301 overflowed CI's 2 MiB stack). + async fn persist_registered_blobs( + &self, + stages: HashMap<(SpaceId, Path), HashBuffer>, + write_hashes: &HashMap<(SpaceId, Path), Hash>, + blobs: &[(SpaceId, Hash, i64)], + upload_deadline: tokio::time::Instant, + ) -> Result<(), TxStoreError> + where + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, + S: ImmutableStaging, + S::Writable: 'static + Unpin, + { + if tokio::time::Instant::now() >= upload_deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "KV upload deadline elapsed before persist", + ) + .into()); + } + tokio::time::timeout_at(upload_deadline, async { + for (key, stage) in stages { + let expected = write_hashes[&key]; + let (space, _) = key; + let index = blobs + .binary_search_by(|(s, v, _)| (s, v).cmp(&(&space, &expected))) + .expect("registered generation for every staged write"); + let hash = self + .storage + .persist(&space, stage, blobs[index].2) + .await + .map_err(TxStoreError::StoreWrite)?; + debug_assert_eq!(hash, expected); + } + Ok::<(), TxStoreError>(()) + }) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "KV upload timed out"))? + } + + /// Perform and record the invocation's side effects inside the KV + /// transaction, after `transact`. Separate from the commit future for the + /// same stack reason as [`Self::persist_registered_blobs`]. + #[allow(clippy::too_many_arguments)] + async fn invocation_side_effects( + &self, + tx: &DatabaseTransaction, + caps: &[crate::util::Capability], + invoker: &str, + options: &KvInvokeOptions, + write_hashes: &HashMap<(SpaceId, Path), Hash>, + deleted_hashes: &HashMap<(SpaceId, Path), Hash>, + caps_read_params: Option<&CapabilitiesReadParams>, + ) -> Result>, TxStoreError> + where + B: ImmutableWriteStore + ImmutableReadStore + ImmutableDeleteStore, + S: ImmutableStaging, + S::Writable: 'static + Unpin, + { + let mut results = Vec::new(); + for cap in caps.iter().filter_map(|c| { + c.resource.tinycloud_resource().and_then(|r| { + Some(( + r.space(), + r.service().as_str(), + // TC-119: resolve deprecated aliases to canonical (see the + // staging loop above) — identity for canonical URNs. + crate::policy_capability::resolve_alias(c.ability.as_ref().as_ref()), + r.path()?, + )) + }) + }) { + match cap { + (space, "kv", "tinycloud.kv/get", path) => { + let data = + get_kv(tx, &self.storage, space, path) + .await + .map_err(|e| match e { + EitherError::A(e) => TxStoreError::Tx(e.into()), + EitherError::B(e) => TxStoreError::StoreRead(e), + })?; + if let (Some(limit), Some((_, _, content))) = + (options.max_response_bytes, data.as_ref()) + { + if content.len() > limit { + return Err(TxStoreError::KvResponseTooLarge { + size: content.len(), + limit, + }); + } + } + results.push(InvocationOutcome::KvRead(data)); + } + (space, "kv", "tinycloud.kv/list", path) => { + let (list, truncated) = list_bounded_after( + tx, + space, + path, + options.list_limit, + options.list_cursor.as_ref(), + ) + .await?; + results.push(InvocationOutcome::KvList(list, truncated, None)) + } + (space, "kv", "tinycloud.kv/del", path) => { + // KV deletion is logical. Blobs are content-addressed and may be + // shared by live sibling keys or retained version history. + results.push(InvocationOutcome::KvDelete( + deleted_hashes.get(&(space.clone(), path.clone())).copied(), + )) + } + (space, "kv", "tinycloud.kv/put", path) => { + let hash = write_hashes + .get(&(space.clone(), path.clone())) + .copied() + .expect("staged KV writes have a content hash"); + results.push(InvocationOutcome::KvWrite(hash)); + } + (space, "kv", "tinycloud.kv/metadata", path) => results.push( + InvocationOutcome::KvMetadata(metadata_with_hash(tx, space, path).await?), + ), + (space, "capabilities", "tinycloud.capabilities/read", path) + if path.as_str() == "all" => + { + match caps_read_params { + None => { + // Backward compatible: no params means return all valid delegations + results.push(InvocationOutcome::OpenSessions( + get_valid_delegations(tx, space, self.encryption.as_ref()).await?, + )) + } + Some(CapabilitiesReadParams::List { filters }) => { + // List with optional filters + results.push(InvocationOutcome::OpenSessions( + get_filtered_delegations( + tx, + space, + invoker, + filters.as_ref(), + self.encryption.as_ref(), + ) + .await?, + )) + } + Some(CapabilitiesReadParams::Chain { delegation_cid }) => { + // Get the delegation chain for a specific delegation + results.push(InvocationOutcome::DelegationChain( + get_delegation_chain( + tx, + space, + delegation_cid, + self.encryption.as_ref(), + ) + .await?, + )) + } + } + } + _ => {} + }; + } + Ok(results) + } + /// Execute a successful non-mutating invocation without entering the /// single-writer transaction. Authorization and data access remain under /// the caller's shared chain guards, which are held across this call and