diff --git a/crates/rds-sync/src/directory.rs b/crates/rds-sync/src/directory.rs new file mode 100644 index 0000000..f7a9045 --- /dev/null +++ b/crates/rds-sync/src/directory.rs @@ -0,0 +1,345 @@ +//! Deterministic directory snapshot model. +//! +//! This module intentionally does not walk the filesystem. A future scanner +//! must provide entries through the confined directory-handle layer so a +//! rename or symlink race cannot turn a convenient `Path` walk into a +//! confinement bypass. The model is useful independently for wire manifests, +//! dry-run previews and conflict planning. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::{ChunkHash, SyncError, proto::check_rel_path}; + +/// Maximum entries accepted in one directory snapshot. +pub const MAX_DIRECTORY_ENTRIES: usize = 1 << 20; +/// Maximum symlink target bytes retained in a snapshot. +pub const MAX_SYMLINK_TARGET_BYTES: usize = 4096; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum EntryKind { + File, + Directory, + Symlink, +} + +/// Content identity and metadata needed by the first one-way mirror model. +/// Unsupported filesystem attributes are deliberately absent rather than +/// silently guessed; the W8 metadata policy will extend this type explicitly. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DirectoryEntry { + /// Canonical UTF-8 relative path, with `/` separators. + pub path: String, + pub kind: EntryKind, + /// File size; zero for directories and symlinks. + pub size: u64, + /// Verified file content root; absent for directories and symlinks. + pub content: Option, + /// Raw symlink target bytes; never followed by this model. + pub symlink_target: Option>, +} + +impl DirectoryEntry { + pub fn file(path: impl Into, size: u64, content: ChunkHash) -> Self { + Self { + path: path.into(), + kind: EntryKind::File, + size, + content: Some(content), + symlink_target: None, + } + } + + pub fn directory(path: impl Into) -> Self { + Self { + path: path.into(), + kind: EntryKind::Directory, + size: 0, + content: None, + symlink_target: None, + } + } + + pub fn symlink(path: impl Into, target: Vec) -> Self { + Self { + path: path.into(), + kind: EntryKind::Symlink, + size: 0, + content: None, + symlink_target: Some(target), + } + } + + fn validate(&self) -> Result<(), SyncError> { + let normalized = check_rel_path(&self.path)?; + if normalized.to_string_lossy() != self.path || self.path.contains('\\') { + return Err(SyncError::Manifest(format!( + "directory entry path is not canonical: {:?}", + self.path + ))); + } + match self.kind { + EntryKind::File if self.content.is_none() || self.symlink_target.is_some() => Err( + SyncError::Manifest("file entry identity is incomplete".into()), + ), + EntryKind::Directory + if self.size != 0 || self.content.is_some() || self.symlink_target.is_some() => + { + Err(SyncError::Manifest( + "directory entry carries file data".into(), + )) + } + EntryKind::Symlink + if self.content.is_some() + || self.symlink_target.as_ref().is_none_or(|target| { + target.is_empty() || target.len() > MAX_SYMLINK_TARGET_BYTES + }) => + { + Err(SyncError::Manifest( + "symlink entry target is invalid".into(), + )) + } + _ => Ok(()), + } + } + + fn identity(&self) -> EntryIdentity { + EntryIdentity { + kind: self.kind, + size: self.size, + content: self.content, + symlink_target: self.symlink_target.clone(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct EntryIdentity { + kind: EntryKind, + size: u64, + content: Option, + symlink_target: Option>, +} + +impl Serialize for EntryIdentity { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + (&self.kind, self.size, self.content, &self.symlink_target).serialize(serializer) + } +} + +/// A canonical, bounded directory snapshot. `root` covers the sorted entries +/// and is independent of the order in which a scanner observed them. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DirectoryManifest { + pub root: ChunkHash, + pub entries: Vec, +} + +impl DirectoryManifest { + pub fn from_entries(mut entries: Vec) -> Result { + if entries.len() > MAX_DIRECTORY_ENTRIES { + return Err(SyncError::Manifest("directory manifest too large".into())); + } + for entry in &entries { + entry.validate()?; + } + entries.sort_by(|a, b| a.path.as_bytes().cmp(b.path.as_bytes())); + if entries.windows(2).any(|pair| pair[0].path == pair[1].path) { + return Err(SyncError::Manifest( + "directory manifest contains duplicate paths".into(), + )); + } + let root = digest_entries(&entries)?; + Ok(Self { root, entries }) + } + + pub fn verify(&self) -> Result<(), SyncError> { + let canonical = Self::from_entries(self.entries.clone())?; + if canonical.root != self.root || canonical.entries != self.entries { + return Err(SyncError::Manifest( + "directory manifest is not canonical".into(), + )); + } + Ok(()) + } + + pub fn diff(&self, other: &Self) -> Result { + self.verify()?; + other.verify()?; + let left: BTreeMap<&str, &DirectoryEntry> = self + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + let right: BTreeMap<&str, &DirectoryEntry> = other + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + let mut added = Vec::new(); + let mut removed = Vec::new(); + let mut modified = Vec::new(); + let mut unchanged = Vec::new(); + for (path, entry) in &right { + match left.get(path) { + None => added.push((*entry).clone()), + Some(previous) if previous.identity() == entry.identity() => { + unchanged.push((*entry).clone()) + } + Some(_) => modified.push((*entry).clone()), + } + } + for (path, entry) in &left { + if !right.contains_key(path) { + removed.push((*entry).clone()); + } + } + let renamed = rename_candidates(&mut added, &mut removed); + Ok(DirectoryDiff { + added, + removed, + modified, + unchanged, + renamed, + }) + } +} + +fn digest_entries(entries: &[DirectoryEntry]) -> Result { + let mut hasher = blake3::Hasher::new(); + for entry in entries { + let bytes = postcard::to_stdvec(entry) + .map_err(|_| SyncError::Manifest("directory entry cannot be encoded".into()))?; + hasher.update(&(bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + } + Ok(*hasher.finalize().as_bytes()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Rename { + pub from: DirectoryEntry, + pub to: DirectoryEntry, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DirectoryDiff { + pub added: Vec, + pub removed: Vec, + pub modified: Vec, + pub unchanged: Vec, + pub renamed: Vec, +} + +fn rename_candidates( + added: &mut Vec, + removed: &mut Vec, +) -> Vec { + let mut by_identity: BTreeMap> = BTreeMap::new(); + for (index, entry) in removed.iter().enumerate() { + by_identity.entry(entry.identity()).or_default().push(index); + } + let mut used = BTreeSet::new(); + let mut renamed = Vec::new(); + for entry in added.iter() { + let Some(indices) = by_identity.get_mut(&entry.identity()) else { + continue; + }; + let Some(index) = indices.pop() else { continue }; + used.insert(index); + renamed.push(Rename { + from: removed[index].clone(), + to: entry.clone(), + }); + } + let mut next_added = Vec::new(); + std::mem::swap(added, &mut next_added); + *added = next_added + .into_iter() + .filter(|entry| !renamed.iter().any(|rename| rename.to.path == entry.path)) + .collect(); + let old_removed = std::mem::take(removed); + *removed = old_removed + .into_iter() + .enumerate() + .filter(|(index, _)| !used.contains(index)) + .map(|(_, entry)| entry) + .collect(); + renamed.sort_by(|a, b| a.to.path.cmp(&b.to.path)); + renamed +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(path: &str, byte: u8) -> DirectoryEntry { + DirectoryEntry::file(path, 1, *blake3::hash(&[byte]).as_bytes()) + } + + #[test] + fn manifest_is_sorted_and_order_independent() { + let a = DirectoryManifest::from_entries(vec![file("z", 1), file("a", 2)]).unwrap(); + let b = DirectoryManifest::from_entries(vec![file("a", 2), file("z", 1)]).unwrap(); + assert_eq!(a, b); + assert!(a.verify().is_ok()); + } + + #[test] + fn traversal_duplicate_and_oversized_entries_fail_closed() { + assert!(DirectoryManifest::from_entries(vec![file("../escape", 1)]).is_err()); + assert!(DirectoryManifest::from_entries(vec![file("a", 1), file("a", 2)]).is_err()); + let entries = (0..=MAX_DIRECTORY_ENTRIES) + .map(|i| file(&format!("{i}"), i as u8)) + .collect(); + assert!(DirectoryManifest::from_entries(entries).is_err()); + } + + #[test] + fn diff_preserves_changes_and_pairs_only_exact_renames() { + let before = + DirectoryManifest::from_entries(vec![file("old", 1), file("same", 7)]).unwrap(); + let after = DirectoryManifest::from_entries(vec![ + file("new", 1), + file("same", 7), + file("changed", 9), + ]) + .unwrap(); + let diff = before.diff(&after).unwrap(); + assert_eq!(diff.renamed.len(), 1); + assert_eq!(diff.renamed[0].from.path, "old"); + assert_eq!(diff.renamed[0].to.path, "new"); + assert_eq!( + diff.unchanged + .iter() + .map(|e| e.path.as_str()) + .collect::>(), + vec!["same"] + ); + assert_eq!( + diff.added + .iter() + .map(|e| e.path.as_str()) + .collect::>(), + vec!["changed"] + ); + } + + #[test] + fn symlink_targets_are_data_and_never_followed() { + let manifest = DirectoryManifest::from_entries(vec![DirectoryEntry::symlink( + "link", + b"../../outside".to_vec(), + )]) + .unwrap(); + assert_eq!(manifest.entries[0].kind, EntryKind::Symlink); + assert_eq!( + manifest.entries[0].symlink_target.as_deref(), + Some(&b"../../outside"[..]) + ); + } +} diff --git a/crates/rds-sync/src/lib.rs b/crates/rds-sync/src/lib.rs index 1629d47..c2d713b 100644 --- a/crates/rds-sync/src/lib.rs +++ b/crates/rds-sync/src/lib.rs @@ -12,6 +12,7 @@ //! transfer protocol — `proto` frames, `journal` resumable state, and //! `engine` driving send/receive/serve over `rds-net` streams. +pub mod directory; pub mod engine; pub mod journal; pub mod proto; diff --git a/docs/architecture.md b/docs/architecture.md index eba41ac..c52f587 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -600,6 +600,13 @@ capability continues to name the same inode after rename; this is not a sandbox against a local process moving already-open directories out of the tree. `Journal::assemble` consumes its journal and takes no new destination root. +The recursive-sync foundation is currently model-only: `rds-sync::directory` +defines bounded canonical file/dir/symlink entries, snapshot roots and +identity-based diff/rename candidates without walking or following a +filesystem. A future scanner must feed that model through the same opened +directory handles before recursive wire or journal behavior is added; the +single-file protocol remains the only advertised sync capability. + Each root has one persistent `receive.lock` inode, locked nonblockingly across processes for a receive's lifetime. A nested destination also holds its parent's private receive lock, so differently configured overlapping roots cannot write diff --git a/docs/completion-plan-20261008.md b/docs/completion-plan-20261008.md index 4805404..5bff429 100644 --- a/docs/completion-plan-20261008.md +++ b/docs/completion-plan-20261008.md @@ -50,7 +50,8 @@ session. Only then change `service:audio` from `stub`. 1. Add a directory manifest format with typed file/dir/symlink metadata, normalized relative paths, deterministic ordering and explicit unsupported - attribute states. Never follow symlinks while scanning. + attribute states. Never follow symlinks while scanning. **Model landed in + `rds-directory-model-20261008`; the no-follow scanner is still open.** 2. Add snapshot/reconcile operations on top of the existing journal, with tombstones, rename detection by stable content identity, bounded entries and cancellation checkpoints. diff --git a/docs/reports/rds-directory-model-20261008.md b/docs/reports/rds-directory-model-20261008.md new file mode 100644 index 0000000..4605dfb --- /dev/null +++ b/docs/reports/rds-directory-model-20261008.md @@ -0,0 +1,22 @@ +# Directory manifest model — 2026-10-08 + +The first W8 wave adds the deterministic directory snapshot model without +pretending that recursive synchronization is already a product capability. + +Implemented in `rds-sync::directory`: + +- bounded entry count and symlink-target size; +- canonical UTF-8 relative paths using the existing traversal and journal + namespace checks; +- explicit file, directory and symlink kinds; +- content identity for files and raw, never-followed symlink targets; +- stable sorting and a snapshot root independent of enumeration order; +- canonical re-verification before diffing; +- deterministic added/removed/modified/unchanged classification and exact + identity-based rename candidates. + +The module does not walk the filesystem, follow links, mutate files or alter +the existing single-file wire protocol. A future scanner must use the existing +directory-handle/no-follow layer, then feed this model. Recursive transfer, +tombstones, conflicts, metadata policy, watch/reconcile and journal GC remain +separate gates.