diff --git a/docs/operations.md b/docs/operations.md index 512f770d..26efdcca 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -236,6 +236,31 @@ next visit asks again. **What it logs.** The provider and a verdict, such as `answered HTTP 503` — never the request URL, which carries your key. +## Scanning + +A scan reads only what changed. A file whose size and modification time are +the ones the catalogue holds, with its `.lrc` and `.txt` lyrics sidecars +unchanged too, is skipped from a `stat` alone, without being opened, so a +rescan of an unchanged library costs a directory walk and not a read of every +byte. The first scan after upgrading to this version reads every file once, to +record what the next ones compare. [RFC-018](rfcs/RFC-018-incremental-scan.md) +carries the reasoning. + +A file rewritten in place with both its size and its modification time kept is +therefore not noticed by an ordinary scan. To have every file read again: + +```bash +waveflow-server database rescan-full +``` + +The next scan of every library then skips nothing, and the request is spent. + +A folder the scan cannot read, such as wrong permissions or an unmounted share, +is reported in the scan's errors with its path relative to the library, and +the scan goes on. The tracks under it are left as they are, not marked missing, +so a folder that is briefly unreadable does not make its albums vanish and come +back. Only the library root itself failing to open fails the scan. + ## Probes | Path | Answers | diff --git a/docs/rfcs/RFC-018-incremental-scan.md b/docs/rfcs/RFC-018-incremental-scan.md index eaad18dc..f0ce68ad 100644 --- a/docs/rfcs/RFC-018-incremental-scan.md +++ b/docs/rfcs/RFC-018-incremental-scan.md @@ -2,7 +2,9 @@ - **Status**: Proposed - **Implemented by**: in progress: [#322](https://github.com/InstaZDLL/waveflow-server/pull/322) - (this RFC). The *Status* field above never flips in this + (this RFC) and [#324](https://github.com/InstaZDLL/waveflow-server/pull/324) + (the pre-check and its sidecar signature, streaming, tolerant discovery, + `database rescan-full`). The *Status* field above never flips in this project; this line is the one that says what is running. - **Date**: 2026-10-24 - **Authors**: the WaveFlow project @@ -47,7 +49,7 @@ hold: - its size and modification time equal the stored ones; - the track is available; - its sidecar signature equals the stored one; -- this is not a full scan (`database rescan --full`, or the identity-spec +- this is not a full scan (`waveflow-server database rescan-full`, or the identity-spec change that forces one); - the track already has the identity hint the skip path writes today, so a row from before that column still gets its one extraction. diff --git a/migrations-v2/20261025010000_track_sidecar_signature.sql b/migrations-v2/20261025010000_track_sidecar_signature.sql new file mode 100644 index 00000000..9156f4c2 --- /dev/null +++ b/migrations-v2/20261025010000_track_sidecar_signature.sql @@ -0,0 +1,11 @@ +-- What an incremental scan compares before it reads a file (RFC-018 decision 1). +-- +-- The size and modification time of the two lyrics sidecars a scan reads beside +-- the audio file, `.lrc` and `.txt`, absent ones included: +-- +-- lrc::|txt:: +-- +-- with `-` for an absent sidecar, so `lrc:-|txt:-` is a file with none. Never +-- NULL once written: NULL means only "never computed", which never matches, so +-- the first scan after this migration reads every file once and fills it. +ALTER TABLE track ADD COLUMN sidecar_signature TEXT; diff --git a/src/catalog.rs b/src/catalog.rs index 303edbc6..d085c886 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -38,6 +38,21 @@ pub struct LibraryAccess { pub accepts_canvas: bool, } +/// What an incremental scan compares before reading a file (RFC-018 +/// decision 1): enough to skip it without opening it. +#[derive(Debug, Clone)] +pub struct SkipFacts { + pub id: Uuid, + pub file_size: i64, + pub modified_at: i64, + pub available: bool, + /// The relocation hint is present; a row from before the column still + /// needs its one extraction. + pub has_pid: bool, + /// `None` until a scan has computed it, which never matches. + pub sidecar_signature: Option, +} + #[derive(Debug, Clone)] pub struct ExistingTrack { pub id: Uuid, @@ -1085,6 +1100,93 @@ impl Database { self.mark_tracks_seen(&[track_id], scan_id).await } + /// Every track of the library by its path, with what the pre-check compares. + /// Read once per scan: a row per track, small, and one query instead of one + /// per file. + pub async fn track_skip_facts( + &self, + library_id: Uuid, + ) -> Result, sqlx::Error> { + let rows = sqlx::query( + "SELECT id, relative_path, file_size, file_modified_at, is_available, pid IS NOT NULL AS has_pid, sidecar_signature FROM track WHERE library_id = ?", + ) + .bind(library_id.to_string()) + .fetch_all(self.pool()) + .await?; + let mut facts = std::collections::HashMap::with_capacity(rows.len()); + for row in rows { + let id: String = row.try_get("id")?; + facts.insert( + row.try_get("relative_path")?, + SkipFacts { + id: Uuid::parse_str(&id).map_err(|error| sqlx::Error::Decode(error.into()))?, + file_size: row.try_get("file_size")?, + modified_at: row.try_get("file_modified_at")?, + available: row.try_get::("is_available")? != 0, + has_pid: row.try_get::("has_pid")? != 0, + sidecar_signature: row.try_get("sidecar_signature")?, + }, + ); + } + Ok(facts) + } + + /// Records the sidecar signature of files a scan read, by path, so the next + /// scan can skip them (RFC-018 decision 1). + pub async fn set_sidecar_signatures( + &self, + library_id: Uuid, + signatures: &[(String, String)], + ) -> Result<(), sqlx::Error> { + if signatures.is_empty() { + return Ok(()); + } + let _writer = self.writer_guard().await; + let mut tx = self.pool().begin().await?; + for (relative_path, signature) in signatures { + sqlx::query( + "UPDATE track SET sidecar_signature = ? WHERE library_id = ? AND relative_path = ?", + ) + .bind(signature) + .bind(library_id.to_string()) + .bind(relative_path) + .execute(&mut *tx) + .await?; + } + tx.commit().await + } + + /// Stamps as seen by this scan the tracks under folders the scan could not + /// read, without touching their availability, so the pass that marks the + /// unseen missing leaves them as they are (RFC-018 decision 3). + pub async fn mark_tracks_under_seen( + &self, + library_id: Uuid, + folders: &[String], + scan_id: Uuid, + ) -> Result<(), sqlx::Error> { + if folders.is_empty() { + return Ok(()); + } + let _writer = self.writer_guard().await; + let mut tx = self.pool().begin().await?; + for folder in folders { + // A prefix compared as text rather than with LIKE: a folder name + // holding `%` or `_` must not widen the match. + sqlx::query( + "UPDATE track SET last_seen_scan_id = ? WHERE library_id = ? AND (relative_path = ? OR substr(relative_path, 1, length(?) + 1) = ? || '/')", + ) + .bind(scan_id.to_string()) + .bind(library_id.to_string()) + .bind(folder) + .bind(folder) + .bind(folder) + .execute(&mut *tx) + .await?; + } + tx.commit().await + } + pub async fn mark_tracks_seen( &self, track_ids: &[Uuid], diff --git a/src/cli.rs b/src/cli.rs index 13fbb3de..3841928b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -306,6 +306,10 @@ pub enum DatabaseCommand { Backup(BackupArgs), /// Restore a backup bundle before opening SQLite. Restore(RestoreArgs), + /// Read every file again on the next scan of every library, skipping + /// nothing (RFC-018): for a file rewritten in place with its size and + /// modification time both kept, which an ordinary scan does not open. + RescanFull, } #[derive(Debug, Args)] @@ -385,6 +389,14 @@ pub async fn execute(command: Command, state: &AppState) -> anyhow::Result<()> { DatabaseCommand::Restore(_) => { anyhow::bail!("restore must run before server initialization") } + DatabaseCommand::RescanFull => { + let libraries = db.request_full_scan_everywhere().await?; + println!( + "The next scan of {libraries} librar{} will read every file.", + if libraries == 1 { "y" } else { "ies" } + ); + Ok(()) + } }, Command::Key { .. } => { anyhow::bail!("a key change must run before server initialization") diff --git a/src/scanner.rs b/src/scanner.rs index 70d1edb0..ce054a69 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -262,7 +262,11 @@ impl ScanManager { let _library_guard = lock.lock().await; tokio::fs::create_dir_all(&self.artwork_dir).await?; let root = tokio::fs::canonicalize(&library.root_path).await?; - let paths = tokio::task::spawn_blocking({ + let Discovery { + paths, + unreadable, + incomplete, + } = tokio::task::spawn_blocking({ let root = root.clone(); move || discover_audio(&root) }) @@ -282,22 +286,33 @@ impl ScanManager { .start_scan_job(scan_id, paths.len() as i64, full) .await?; + // What the catalogue already knows, read once: the pre-check compares + // against it before a file is opened (RFC-018 decision 1). + let facts = Arc::new(self.db.track_skip_facts(library.id).await?); let root_for_tasks = root.clone(); let artwork_dir = self.artwork_dir.clone(); - let mut extracted = stream::iter(paths.into_iter().map(|path| { + // In path order, as they finish, a few batches at most in memory + // (RFC-018 decision 2): `buffered` keeps the order `paths` has, so the + // outcome is the same however the extractions interleave. + let mut scanned = stream::iter(paths.into_iter().map(|path| { let root = root_for_tasks.clone(); let artwork = artwork_dir.clone(); + let facts = Arc::clone(&facts); async move { let relative = relative_path(&root, &path); + let known = relative + .as_ref() + .ok() + .and_then(|relative| facts.get(relative)) + .cloned(); let result = - tokio::task::spawn_blocking(move || extract_file(&path, &artwork)).await; + tokio::task::spawn_blocking(move || scan_file(&path, &artwork, known, full)) + .await; (relative, result) } })) - .buffer_unordered(self.parallelism) - .collect::>() - .await; - extracted.sort_by(|left, right| scan_result_path(left).cmp(scan_result_path(right))); + .buffered(self.parallelism) + .chunks(25); let mut progress = ScanProgress { scan_id, @@ -315,20 +330,62 @@ impl ScanManager { message: None, }; - for batch in extracted.chunks(25) { + // A folder the walk could not read is reported, and its tracks are left + // as they are below (RFC-018 decision 3). + if !unreadable.is_empty() { + progress.errors += unreadable.len() as i64; + let errors: Vec<(String, String)> = unreadable + .iter() + .map(|folder| (folder.clone(), "folder could not be read".to_owned())) + .collect(); + self.db.add_scan_errors(scan_id, &errors).await?; + // Written now, not left to the batches: a library whose only + // content sits under the unreadable folder has no batch at all. + self.tick(&progress).await?; + } + + let walked = Walked { + discovered: &discovered, + unreadable: &unreadable, + incomplete, + }; + while let Some(batch) = scanned.next().await { let mut applies = Vec::new(); let mut seen = Vec::new(); let mut errors = Vec::new(); let mut claimed_relocations = HashSet::new(); let mut hints = Vec::new(); + let mut signatures = Vec::new(); for (relative, task_result) in batch { progress.processed += 1; + let task_result = match task_result { + // Unchanged, and never opened. + Ok(Scanned::Skipped(id)) => { + seen.push(id); + progress.skipped += 1; + continue; + } + Ok(Scanned::Read(result, signature)) => { + // Only a read that succeeded records what it read: a + // failed one must be tried again, sidecars included. + if let (Ok(relative), Ok(_), Some(signature)) = + (&relative, result.as_ref(), signature) + { + signatures.push((relative.clone(), signature)); + } + Ok(*result) + } + Err(error) => Err(error), + }; match (relative, task_result) { (Ok(relative), Ok(Ok(input))) => { - let mut input = input.clone(); + let mut input = input; progress.current_path = Some(relative.clone()); input.relative_path = relative.clone(); - let existing = self.db.existing_track_by_path(library.id, relative).await?; + let existing = self + .db + .existing_track_by_path(library.id, &relative) + .await?; if let Some(existing) = existing.as_ref() { if !full && existing.file_size == input.file_size @@ -367,7 +424,7 @@ impl ScanManager { self.db .relocation_candidates(library.id, &input.full_hash) .await?, - &discovered, + &walked, &claimed_relocations, ); if missing.is_empty() { @@ -387,7 +444,7 @@ impl ScanManager { ), ) .await?, - &discovered, + &walked, &claimed_relocations, ); } @@ -409,12 +466,9 @@ impl ScanManager { } (relative, result) => { progress.errors += 1; - let path = relative - .as_ref() - .cloned() - .unwrap_or_else(|_| "".into()); + let path = relative.unwrap_or_else(|_| "".into()); let message = match result { - Ok(Err(error)) => error.clone(), + Ok(Err(error)) => error, Err(error) => format!("extract worker failed: {error}"), Ok(Ok(_)) => "relative path failed".into(), }; @@ -436,11 +490,27 @@ impl ScanManager { ApplyOutcome::Moved => progress.moved += 1, } } + self.db + .set_sidecar_signatures(library.id, &signatures) + .await?; self.db.add_scan_errors(scan_id, &errors).await?; self.tick(&progress).await?; } - progress.unavailable = self.db.mark_unseen_unavailable(library.id, scan_id).await?; + self.db + .mark_tracks_under_seen(library.id, &unreadable, scan_id) + .await?; + // A walk error that named no path leaves no folder to spare, so + // nothing this walk missed can be called missing. + progress.unavailable = if incomplete { + tracing::warn!( + library_id = %library.id, + "the walk could not say what it failed to read; nothing is marked missing this scan" + ); + 0 + } else { + self.db.mark_unseen_unavailable(library.id, scan_id).await? + }; // After availability is settled, not before: the album and artist // identifiers are a majority vote over the tracks that are still there, // so a file that vanished must stop voting first. @@ -484,23 +554,158 @@ impl ScanManager { } } -fn scan_result_path( - result: &( - anyhow::Result, - Result, tokio::task::JoinError>, - ), -) -> &str { - result.0.as_deref().unwrap_or("") -} - fn scan_id(progress: &ScanProgress) -> Uuid { progress.scan_id } -fn discover_audio(root: &Path) -> anyhow::Result> { +/// What a scan does with one discovered file. +enum Scanned { + /// Unchanged since the last scan: skipped without being opened. + Skipped(Uuid), + /// Read, with the sidecar signature to record once it is applied. + /// `None` when a sidecar could not be opened: nothing is recorded, so the + /// file is read again until it can be. + Read(Box>, Option), +} + +/// The pre-check, then the read (RFC-018 decision 1). +/// +/// A file whose size, modification time and sidecar signature match what the +/// catalogue holds, on an available track that already has its relocation +/// hint, is skipped from a `stat` alone. Anything else, and every file on a +/// full scan, is read as before. +fn scan_file( + path: &Path, + artwork_dir: &Path, + known: Option, + full: bool, +) -> Scanned { + let signature = sidecar_signature(path); + if let Some(known) = known.filter(|_| !full) { + let unchanged = file_stamp(path).is_some_and(|(size, modified_at)| { + size == known.file_size && modified_at == known.modified_at + }) && known.available + && known.has_pid + && signature.is_some() + && known.sidecar_signature == signature; + if unchanged { + return Scanned::Skipped(known.id); + } + } + Scanned::Read(Box::new(extract_file(path, artwork_dir)), signature) +} + +/// A file's size and modification time, as `extract_file` records them. +fn file_stamp(path: &Path) -> Option<(i64, i64)> { + let metadata = fs::metadata(path).ok()?; + let modified_at = metadata + .modified() + .ok() + .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) + .map(|value| value.as_millis() as i64) + .unwrap_or(0); + Some((metadata.len() as i64, modified_at)) +} + +/// The two lyrics sidecars a scan reads beside an audio file, by size and +/// modification time, absent ones as `-`: +/// `lrc::|txt::`. Adding, editing or deleting +/// either changes it; it is never empty. +/// +/// `None` when a sidecar exists but cannot be opened. The read then goes ahead +/// without its lyrics, and recording the signature would skip the file for as +/// long as the sidecar's size and time stay put — a permission fixed later +/// moves neither. With no signature the file is read on every scan until the +/// sidecar can be. +/// +/// Read without following links, and a sidecar that is not a regular file — a +/// link, a FIFO, a folder — counts as absent, since `read_sidecar` would not +/// read it either. Only a regular file is opened, and never in a way that can +/// block. +fn sidecar_signature(path: &Path) -> Option { + let mut parts = Vec::with_capacity(2); + for extension in ["lrc", "txt"] { + let sidecar = path.with_extension(extension); + match fs::symlink_metadata(&sidecar) { + Ok(metadata) if metadata.is_file() => { + sidecar_options().open(&sidecar).ok()?; + let modified_at = metadata + .modified() + .ok() + .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) + .map(|value| value.as_millis() as i64) + .unwrap_or(0); + parts.push(format!("{extension}:{}:{modified_at}", metadata.len())); + } + _ => parts.push(format!("{extension}:-")), + } + } + Some(parts.join("|")) +} + +/// How a sidecar is opened, for reading it and for checking that it can be: +/// never through a link, and without blocking on a special file swapped in +/// after it was looked at. +fn sidecar_options() -> fs::OpenOptions { + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + // Open the reparse point itself so descriptor metadata can reject a + // symlink instead of validating one path and reading its later target. + const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; + options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT); + } + options +} + +/// What the walk found: the audio files, in path order, and the folders it +/// could not read, relative to the root. +struct Discovery { + paths: Vec, + unreadable: Vec, + /// An error named no path inside the root, so the walk cannot say which + /// tracks it missed: none may be marked missing. + incomplete: bool, +} + +/// Walks the library (RFC-018 decision 3). An entry the walk cannot read is +/// logged by its path relative to the root, never the absolute one, and the +/// walk goes on; only the root itself failing fails the scan, since then +/// nothing can be told apart. +fn discover_audio(root: &Path) -> anyhow::Result { let mut paths = Vec::new(); + let mut unreadable = Vec::new(); + let mut incomplete = false; for entry in WalkDir::new(root).follow_links(false) { - let entry = entry?; + let entry = match entry { + Ok(entry) => entry, + Err(error) if error.depth() == 0 => return Err(error.into()), + Err(error) => { + let Some(folder) = error + .path() + .and_then(|path| path.strip_prefix(root).ok()) + .map(|path| path.to_string_lossy().replace('\\', "/")) + else { + tracing::warn!("the library walk failed somewhere it could not name"); + incomplete = true; + unreadable.push("".to_owned()); + continue; + }; + tracing::warn!( + path = %folder, + "a folder in the library could not be read; its tracks are left as they are" + ); + unreadable.push(folder); + continue; + } + }; if entry.file_type().is_symlink() || !entry.file_type().is_file() { continue; } @@ -518,25 +723,54 @@ fn discover_audio(root: &Path) -> anyhow::Result> { } } paths.sort(); - Ok(paths) + Ok(Discovery { + paths, + unreadable, + incomplete, + }) +} + +/// What one walk saw, as relocation needs it: the files it found, and what +/// it could not look at. +struct Walked<'a> { + discovered: &'a HashSet, + unreadable: &'a [String], + incomplete: bool, +} + +impl Walked<'_> { + /// Whether the walk can say this track's file is gone: it was not found, + /// and it does not sit under a folder the walk could not read. + fn saw_missing(&self, relative_path: &str) -> bool { + !self.incomplete + && !self.discovered.contains(relative_path) + && !self.unreadable.iter().any(|folder| { + relative_path == folder + || relative_path + .strip_prefix(folder.as_str()) + .is_some_and(|rest| rest.starts_with('/')) + }) + } } /// The candidates a scan may still claim. /// /// A track whose file this walk found is not missing, whatever a hash or a hint /// says about it, and one already claimed by an earlier file in this batch is -/// spoken for. What survives is what genuinely disappeared — and a relocation -/// is only recorded when exactly one thing did, because moving a favourite onto -/// the wrong row is worse than not moving it at all. +/// spoken for. Nor is one under a folder the walk could not read, or any at all +/// when the walk could not say what it missed (RFC-018 decision 3): its file +/// may be right where it was. What survives is what genuinely disappeared — and +/// a relocation is only recorded when exactly one thing did, because moving a +/// favourite onto the wrong row is worse than not moving it at all. fn unclaimed( candidates: Vec, - discovered: &HashSet, + walked: &Walked<'_>, claimed: &HashSet, ) -> Vec { candidates .into_iter() .filter(|candidate| { - !discovered.contains(&candidate.relative_path) && !claimed.contains(&candidate.id) + walked.saw_missing(&candidate.relative_path) && !claimed.contains(&candidate.id) }) .collect() } @@ -994,22 +1228,7 @@ fn push_lyrics(found: &mut Vec, source: &'static str, content: &str } fn read_sidecar(path: &Path) -> Option { - let mut options = fs::OpenOptions::new(); - options.read(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.custom_flags(libc::O_NOFOLLOW); - } - #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt; - // Open the reparse point itself so descriptor metadata can reject a - // symlink instead of validating one path and reading its later target. - const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; - options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT); - } - let file = options.open(path).ok()?; + let file = sidecar_options().open(path).ok()?; let metadata = file.metadata().ok()?; if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() > MAX_LYRICS_BYTES { @@ -1044,7 +1263,31 @@ fn lyrics_hash(entries: &[LyricsInput]) -> String { #[cfg(test)] mod tests { - use super::{extended_tags, read_sidecar, MAX_LYRICS_BYTES}; + use super::{extended_tags, read_sidecar, Walked, MAX_LYRICS_BYTES}; + + /// A track under a folder the walk could not read is not missing, so no + /// file elsewhere may claim it; nor may any track when the walk could not + /// name what it missed. + #[test] + fn a_track_the_walk_could_not_see_is_not_missing() { + let discovered = std::collections::HashSet::from(["Found/One.wav".to_owned()]); + let unreadable = vec!["Locked".to_owned()]; + let walked = Walked { + discovered: &discovered, + unreadable: &unreadable, + incomplete: false, + }; + assert!(!walked.saw_missing("Found/One.wav")); + assert!(!walked.saw_missing("Locked/Two.wav")); + assert!(!walked.saw_missing("Locked")); + assert!(walked.saw_missing("Lockedness/Three.wav")); + assert!(walked.saw_missing("Gone/Four.wav")); + let incomplete = Walked { + incomplete: true, + ..walked + }; + assert!(!incomplete.saw_missing("Gone/Four.wav")); + } use lofty::{ prelude::ItemKey, tag::{Tag, TagType}, diff --git a/tests/cli.rs b/tests/cli.rs index f1fc7359..0b7f7c98 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1164,3 +1164,30 @@ async fn a_key_change_prints_what_it_costs_and_then_what_it_did() { "the adoption left no key behind" ); } + +/// RFC-018: an incremental scan skips a file whose size and modification time +/// are unchanged, so the operator needs a way to have every file read again. +/// The request is stored on every library and spent by its next scan. +#[tokio::test] +async fn database_rescan_full_asks_every_library_to_read_everything() { + let (_temp, config, state) = test_app().await; + let owner = inserted_account(&state, "rescan-owner", "admin").await; + let music = config.data_dir.join("rescan-music"); + std::fs::create_dir_all(&music).unwrap(); + let library = state + .db + .create_library( + owner, + "Rescan", + &std::fs::canonicalize(&music).unwrap(), + waveflow_server::database::LibraryVisibility::Private, + waveflow_server::authentication::now_ms(), + ) + .await + .unwrap(); + assert!(!state.db.full_scan_requested(library).await.unwrap()); + + run_cli(&state, &["database", "rescan-full"]).await.unwrap(); + + assert!(state.db.full_scan_requested(library).await.unwrap()); +} diff --git a/tests/scanner.rs b/tests/scanner.rs index 9e7d153b..c0e0f87d 100644 --- a/tests/scanner.rs +++ b/tests/scanner.rs @@ -1080,3 +1080,367 @@ fn scan_progress_converts_into_the_shape_the_stream_sends() { ] ); } + +// ── An incremental scan that reads only what changed (RFC-018) ── + +/// A library of one owner's, holding the files `files` writes into its root. +async fn incremental_library( + name: &str, + files: impl FnOnce(&std::path::Path), +) -> ( + tempfile::TempDir, + waveflow_server::AppState, + uuid::Uuid, + LibraryRecord, + std::path::PathBuf, +) { + let (temp, config, state) = test_app().await; + // Inserted rather than created: a scan needs an owner, not a login. + let owner = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO account (id, username, password_hash, role, disabled, created_at, updated_at) \ + VALUES (?, ?, 'this-account-never-authenticates', 'admin', 0, ?, ?)", + ) + .bind(owner.to_string()) + .bind(name) + .bind(now_ms()) + .bind(now_ms()) + .execute(state.db.pool()) + .await + .unwrap(); + let music = config.data_dir.join(name); + std::fs::create_dir_all(&music).unwrap(); + files(&music); + let root = std::fs::canonicalize(&music).unwrap(); + let library_id = state + .db + .create_library(owner, name, &root, LibraryVisibility::Private, now_ms()) + .await + .unwrap(); + let library = LibraryRecord { + id: library_id, + name: name.into(), + root_path: root.clone(), + }; + (temp, state, owner, library, root) +} + +async fn job( + state: &waveflow_server::AppState, + owner: uuid::Uuid, + scan: uuid::Uuid, +) -> waveflow_server::catalog::ScanJobRecord { + state + .db + .scan_job_for_user(owner, scan) + .await + .unwrap() + .unwrap() +} + +async fn full_hash(state: &waveflow_server::AppState) -> Option { + sqlx::query_scalar("SELECT full_hash FROM track") + .fetch_one(state.db.pool()) + .await + .unwrap() +} + +async fn sidecar_signature(state: &waveflow_server::AppState) -> Option { + sqlx::query_scalar("SELECT sidecar_signature FROM track") + .fetch_one(state.db.pool()) + .await + .unwrap() +} + +/// Rewrites `path` in place with different bytes of the same length, then +/// puts its modification time back: a change only reading the file can see. +fn rewrite_keeping_size_and_time(path: &std::path::Path) { + let modified = std::fs::metadata(path).unwrap().modified().unwrap(); + let mut bytes = std::fs::read(path).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 0x5a; + std::fs::write(path, &bytes).unwrap(); + let file = std::fs::File::options().write(true).open(path).unwrap(); + file.set_times(std::fs::FileTimes::new().set_modified(modified)) + .unwrap(); +} + +/// RFC-018 decision 1: a file whose size, modification time and sidecars are +/// unchanged is skipped without being read. A change only its bytes show — +/// same size, same time — therefore goes unseen until a full scan reads it, +/// which is the trade the RFC states. +#[tokio::test] +async fn an_unchanged_file_is_skipped_without_being_read() { + let (_temp, state, owner, library, root) = incremental_library("incremental-skip", |music| { + write_test_wav(&music.join("Only.wav")) + }) + .await; + let first = scan_once(&state, owner, library.clone()).await; + assert_eq!(job(&state, owner, first).await.added, 1); + let hash = full_hash(&state).await; + + rewrite_keeping_size_and_time(&root.join("Only.wav")); + let second = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!(second.skipped, 1); + assert_eq!(second.updated, 0); + assert_eq!( + full_hash(&state).await, + hash, + "the file was read: its new hash was recorded" + ); + + state.db.request_full_scan_everywhere().await.unwrap(); + let full = job(&state, owner, scan_once(&state, owner, library).await).await; + assert_eq!(full.updated, 1); + assert_ne!(full_hash(&state).await, hash); +} + +/// A sidecar added, edited or removed beside an unchanged audio file is still +/// read: the signature it is compared on covers both sidecars, absent ones +/// included. +#[tokio::test] +async fn a_sidecar_changed_beside_an_unchanged_file_is_read() { + let (_temp, state, owner, library, root) = + incremental_library("incremental-sidecar", |music| { + write_test_wav(&music.join("Song.wav")) + }) + .await; + scan_once(&state, owner, library.clone()).await; + assert_eq!( + sidecar_signature(&state).await.as_deref(), + Some("lrc:-|txt:-") + ); + + std::fs::write(root.join("Song.lrc"), "[00:01.00]A line\n").unwrap(); + let added = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!(added.skipped, 0, "the new sidecar was not read"); + assert_eq!(added.updated, 1); + + let unchanged = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!(unchanged.skipped, 1); + + std::fs::remove_file(root.join("Song.lrc")).unwrap(); + let removed = job(&state, owner, scan_once(&state, owner, library).await).await; + assert_eq!(removed.skipped, 0, "the removed sidecar went unnoticed"); + assert_eq!(removed.updated, 1); +} + +/// `NULL` means never computed and never matches: right after the migration +/// every file is read once, which fills the signature, and the scan after it +/// skips. +#[tokio::test] +async fn a_signature_never_computed_is_filled_by_the_next_scan() { + let (_temp, state, owner, library, _root) = incremental_library("incremental-null", |music| { + write_test_wav(&music.join("Only.wav")) + }) + .await; + scan_once(&state, owner, library.clone()).await; + sqlx::query("UPDATE track SET sidecar_signature = NULL") + .execute(state.db.pool()) + .await + .unwrap(); + + scan_once(&state, owner, library.clone()).await; + assert_eq!( + sidecar_signature(&state).await.as_deref(), + Some("lrc:-|txt:-"), + "an unchanged file read for want of a signature must record one" + ); + let next = job(&state, owner, scan_once(&state, owner, library).await).await; + assert_eq!(next.skipped, 1); +} + +/// RFC-018 decision 3: a folder the walk cannot read costs that folder, not +/// the scan, and its tracks are not marked missing. Unix only: it is the one +/// place a test can take a folder's permissions away. +#[cfg(unix)] +#[tokio::test] +async fn an_unreadable_folder_costs_that_folder_only() { + use std::os::unix::fs::PermissionsExt; + + let (_temp, state, owner, library, root) = + incremental_library("incremental-unreadable", |music| { + std::fs::create_dir_all(music.join("Readable")).unwrap(); + std::fs::create_dir_all(music.join("Locked")).unwrap(); + write_test_wav(&music.join("Readable/One.wav")); + write_test_wav_of_len(&music.join("Locked/Two.wav"), 900); + }) + .await; + let first = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!(first.added, 2); + + /// Puts the permissions back however the test ends, or the temporary + /// directory could not be removed. + struct Unlock(std::path::PathBuf); + impl Drop for Unlock { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o755)); + } + } + + let locked = root.join("Locked"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap(); + let unlock = Unlock(locked.clone()); + if std::fs::read_dir(&locked).is_ok() { + // Root reads through `chmod 000`: nothing here can be made unreadable. + eprintln!("skipped: the folder stays readable to this user"); + return; + } + let second = scan_once(&state, owner, library.clone()).await; + drop(unlock); + + let second = job(&state, owner, second).await; + assert_eq!(second.errors, 1); + assert_eq!( + second.unavailable, 0, + "a track under the unreadable folder was marked missing" + ); + let available: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM track WHERE is_available = 1") + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(available, 2); +} + +/// When everything a library holds sits under the unreadable folder, the walk +/// finds no file and the scan runs no batch: the folder's error must still be +/// counted on the job, and its track still not marked missing. +#[cfg(unix)] +#[tokio::test] +async fn an_unreadable_folder_holding_everything_is_still_counted() { + use std::os::unix::fs::PermissionsExt; + + struct Unlock(std::path::PathBuf); + impl Drop for Unlock { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o755)); + } + } + + let (_temp, state, owner, library, root) = + incremental_library("incremental-all-locked", |music| { + std::fs::create_dir_all(music.join("Locked")).unwrap(); + write_test_wav(&music.join("Locked/Only.wav")); + }) + .await; + assert_eq!( + job( + &state, + owner, + scan_once(&state, owner, library.clone()).await + ) + .await + .added, + 1 + ); + + let locked = root.join("Locked"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap(); + let unlock = Unlock(locked.clone()); + if std::fs::read_dir(&locked).is_ok() { + eprintln!("skipped: the folder stays readable to this user"); + return; + } + let second = scan_once(&state, owner, library).await; + drop(unlock); + + let second = job(&state, owner, second).await; + assert_eq!( + second.errors, 1, + "the folder's error was not recorded on the job" + ); + assert_eq!(second.unavailable, 0); +} + +/// A sidecar the scan can list but not open is read past, without its lyrics, +/// and no signature is recorded: once the permission is fixed — which moves +/// neither its size nor its time — the next scan reads it. Unix only, for the +/// permissions. +#[cfg(unix)] +#[tokio::test] +async fn a_sidecar_that_cannot_be_opened_is_read_again_once_it_can() { + use std::os::unix::fs::PermissionsExt; + + struct Unlock(std::path::PathBuf); + impl Drop for Unlock { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o644)); + } + } + + let (_temp, state, owner, library, root) = + incremental_library("incremental-locked-lyrics", |music| { + write_test_wav(&music.join("Song.wav")); + std::fs::write(music.join("Song.lrc"), "[00:01.00]A line\n").unwrap(); + }) + .await; + let sidecar = root.join("Song.lrc"); + std::fs::set_permissions(&sidecar, std::fs::Permissions::from_mode(0o000)).unwrap(); + let unlock = Unlock(sidecar.clone()); + if std::fs::File::open(&sidecar).is_ok() { + eprintln!("skipped: the sidecar stays readable to this user"); + return; + } + scan_once(&state, owner, library.clone()).await; + assert_eq!(sidecar_signature(&state).await, None); + + drop(unlock); + let again = job(&state, owner, scan_once(&state, owner, library).await).await; + assert_eq!(again.skipped, 0, "the now-readable sidecar was never read"); + assert_eq!(again.updated, 1); + assert!(sidecar_signature(&state).await.is_some()); +} + +/// A sidecar that is a FIFO with no writer does not hang the scan: the +/// signature never opens anything but a regular file, and the read never +/// blocks. `scan_once` gives up after two seconds, so a hang fails here. +#[cfg(unix)] +#[tokio::test] +async fn a_fifo_beside_an_audio_file_does_not_hang_the_scan() { + let (_temp, state, owner, library, _root) = incremental_library("incremental-fifo", |music| { + write_test_wav(&music.join("Song.wav")); + let fifo = std::ffi::CString::new( + music + .join("Song.lrc") + .to_str() + .expect("a UTF-8 temporary path"), + ) + .unwrap(); + // SAFETY: a valid, NUL-terminated path and an ordinary mode. + assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o644) }, 0); + }) + .await; + let first = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!(first.added, 1); + assert_eq!( + sidecar_signature(&state).await.as_deref(), + Some("lrc:-|txt:-") + ); + let second = job(&state, owner, scan_once(&state, owner, library).await).await; + assert_eq!(second.skipped, 1); +}