feat: an incremental scan that reads only what changed (RFC-018)

docs/operations.md

@@ -236,6 +236,31 @@ next visit asks again.
 **What it logs.** The provider and a verdict, such as `answered HTTP 503` —
 never the request URL, which carries your key.
 
+## Scanning
+
+A scan reads only what changed. A file whose size and modification time are
+the ones the catalogue holds, with its `.lrc` and `.txt` lyrics sidecars
+unchanged too, is skipped from a `stat` alone, without being opened, so a
+rescan of an unchanged library costs a directory walk and not a read of every
+byte. The first scan after upgrading to this version reads every file once, to
+record what the next ones compare. [RFC-018](rfcs/RFC-018-incremental-scan.md)
+carries the reasoning.
+
+A file rewritten in place with both its size and its modification time kept is
+therefore not noticed by an ordinary scan. To have every file read again:
+
+```bash
+waveflow-server database rescan-full
+```
+
+The next scan of every library then skips nothing, and the request is spent.
+
+A folder the scan cannot read, such as wrong permissions or an unmounted share,
+is reported in the scan's errors with its path relative to the library, and
+the scan goes on. The tracks under it are left as they are, not marked missing,
+so a folder that is briefly unreadable does not make its albums vanish and come
+back. Only the library root itself failing to open fails the scan.
+
 ## Probes
 
 | Path | Answers |

docs/rfcs/RFC-018-incremental-scan.md

@@ -2,7 +2,9 @@
 
 - **Status**: Proposed
 - **Implemented by**: in progress: [#322](https://github.com/InstaZDLL/waveflow-server/pull/322)
-  (this RFC). The *Status* field above never flips in this
+  (this RFC) and [#324](https://github.com/InstaZDLL/waveflow-server/pull/324)
+  (the pre-check and its sidecar signature, streaming, tolerant discovery,
+  `database rescan-full`). The *Status* field above never flips in this
   project; this line is the one that says what is running.
 - **Date**: 2026-10-24
 - **Authors**: the WaveFlow project
@@ -47,7 +49,7 @@ hold:
 - its size and modification time equal the stored ones;
 - the track is available;
 - its sidecar signature equals the stored one;
-- this is not a full scan (`database rescan --full`, or the identity-spec
+- this is not a full scan (`waveflow-server database rescan-full`, or the identity-spec
   change that forces one);
 - the track already has the identity hint the skip path writes today, so a row
   from before that column still gets its one extraction.

migrations-v2/20261025010000_track_sidecar_signature.sql

@@ -0,0 +1,11 @@
+-- What an incremental scan compares before it reads a file (RFC-018 decision 1).
+--
+-- The size and modification time of the two lyrics sidecars a scan reads beside
+-- the audio file, `<name>.lrc` and `<name>.txt`, absent ones included:
+--
+--     lrc:<size>:<mtime_ms>|txt:<size>:<mtime_ms>
+--
+-- with `-` for an absent sidecar, so `lrc:-|txt:-` is a file with none. Never
+-- NULL once written: NULL means only "never computed", which never matches, so
+-- the first scan after this migration reads every file once and fills it.
+ALTER TABLE track ADD COLUMN sidecar_signature TEXT;

src/catalog.rs

@@ -38,6 +38,21 @@ pub struct LibraryAccess {
     pub accepts_canvas: bool,
 }
 
+/// What an incremental scan compares before reading a file (RFC-018
+/// decision 1): enough to skip it without opening it.
+#[derive(Debug, Clone)]
+pub struct SkipFacts {
+    pub id: Uuid,
+    pub file_size: i64,
+    pub modified_at: i64,
+    pub available: bool,
+    /// The relocation hint is present; a row from before the column still
+    /// needs its one extraction.
+    pub has_pid: bool,
+    /// `None` until a scan has computed it, which never matches.
+    pub sidecar_signature: Option<String>,
+}
+
 #[derive(Debug, Clone)]
 pub struct ExistingTrack {
     pub id: Uuid,
@@ -1085,6 +1100,93 @@ impl Database {
         self.mark_tracks_seen(&[track_id], scan_id).await
     }
 
+    /// Every track of the library by its path, with what the pre-check compares.
+    /// Read once per scan: a row per track, small, and one query instead of one
+    /// per file.
+    pub async fn track_skip_facts(
+        &self,
+        library_id: Uuid,
+    ) -> Result<std::collections::HashMap<String, SkipFacts>, sqlx::Error> {
+        let rows = sqlx::query(
+            "SELECT id, relative_path, file_size, file_modified_at, is_available,                     pid IS NOT NULL AS has_pid, sidecar_signature              FROM track WHERE library_id = ?",
+        )
+        .bind(library_id.to_string())
+        .fetch_all(self.pool())
+        .await?;
+        let mut facts = std::collections::HashMap::with_capacity(rows.len());
+        for row in rows {
+            let id: String = row.try_get("id")?;
+            facts.insert(
+                row.try_get("relative_path")?,
+                SkipFacts {
+                    id: Uuid::parse_str(&id).map_err(|error| sqlx::Error::Decode(error.into()))?,
+                    file_size: row.try_get("file_size")?,
+                    modified_at: row.try_get("file_modified_at")?,
+                    available: row.try_get::<i64, _>("is_available")? != 0,
+                    has_pid: row.try_get::<i64, _>("has_pid")? != 0,
+                    sidecar_signature: row.try_get("sidecar_signature")?,
+                },
+            );
+        }
+        Ok(facts)
+    }
+
+    /// Records the sidecar signature of files a scan read, by path, so the next
+    /// scan can skip them (RFC-018 decision 1).
+    pub async fn set_sidecar_signatures(
+        &self,
+        library_id: Uuid,
+        signatures: &[(String, String)],
+    ) -> Result<(), sqlx::Error> {
+        if signatures.is_empty() {
+            return Ok(());
+        }
+        let _writer = self.writer_guard().await;
+        let mut tx = self.pool().begin().await?;
+        for (relative_path, signature) in signatures {
+            sqlx::query(
+                "UPDATE track SET sidecar_signature = ? WHERE library_id = ? AND relative_path = ?",
+            )
+            .bind(signature)
+            .bind(library_id.to_string())
+            .bind(relative_path)
+            .execute(&mut *tx)
+            .await?;
+        }
+        tx.commit().await
+    }
+
+    /// Stamps as seen by this scan the tracks under folders the scan could not
+    /// read, without touching their availability, so the pass that marks the
+    /// unseen missing leaves them as they are (RFC-018 decision 3).
+    pub async fn mark_tracks_under_seen(
+        &self,
+        library_id: Uuid,
+        folders: &[String],
+        scan_id: Uuid,
+    ) -> Result<(), sqlx::Error> {
+        if folders.is_empty() {
+            return Ok(());
+        }
+        let _writer = self.writer_guard().await;
+        let mut tx = self.pool().begin().await?;
+        for folder in folders {
+            // A prefix compared as text rather than with LIKE: a folder name
+            // holding `%` or `_` must not widen the match.
+            sqlx::query(
+                "UPDATE track SET last_seen_scan_id = ?                  WHERE library_id = ?                    AND (relative_path = ? OR substr(relative_path, 1, length(?) + 1) = ? || '/')",
+            )
+            .bind(scan_id.to_string())
+            .bind(library_id.to_string())
+            .bind(folder)
+            .bind(folder)
+            .bind(folder)
+            .execute(&mut *tx)
+            .await?;
+        }
+        tx.commit().await
+    }
+
     pub async fn mark_tracks_seen(
         &self,
         track_ids: &[Uuid],

src/cli.rs

@@ -306,6 +306,10 @@ pub enum DatabaseCommand {
     Backup(BackupArgs),
     /// Restore a backup bundle before opening SQLite.
     Restore(RestoreArgs),
+    /// Read every file again on the next scan of every library, skipping
+    /// nothing (RFC-018): for a file rewritten in place with its size and
+    /// modification time both kept, which an ordinary scan does not open.
+    RescanFull,
 }
 
 #[derive(Debug, Args)]
@@ -385,6 +389,14 @@ pub async fn execute(command: Command, state: &AppState) -> anyhow::Result<()> {
             DatabaseCommand::Restore(_) => {
                 anyhow::bail!("restore must run before server initialization")
             }
+            DatabaseCommand::RescanFull => {
+                let libraries = db.request_full_scan_everywhere().await?;
+                println!(
+                    "The next scan of {libraries} librar{} will read every file.",
+                    if libraries == 1 { "y" } else { "ies" }
+                );
+                Ok(())
+            }
         },
         Command::Key { .. } => {
             anyhow::bail!("a key change must run before server initialization")