feat: stop PictoPy's own file writes from triggering a folder resync

backend/app/database/self_writes.py

@@ -0,0 +1,154 @@
+"""
+A short-lived record of files PictoPy modified itself.
+
+The sync microservice watches every registered folder and treats any file change
+as a user edit, so a write of our own comes straight back as a full folder
+resync. Recording what we wrote lets the watcher tell the two apart.
+
+The path format here is a contract with `sync-microservice/app/database/
+self_writes.py`, which reads this table: both sides key on
+`os.path.normcase(os.path.abspath(path))`.
+"""
+
+import os
+import sqlite3
+import time
+from typing import List, Set, Tuple
+
+from app.config.settings import DATABASE_PATH
+from app.logging.setup_logging import get_logger
+
+logger = get_logger(__name__)
+
+# An entry the watcher never claims is one it missed -- it was stopped, or the
+# change was coalesced away. Expiring them keeps the table bounded and limits
+# how long a stale row can mask a real edit to the same path.
+SELF_WRITE_TTL_SECONDS = 3600
+
+# (path, file_size, file_mtime) as observed on disk right now.
+ObservedFile = Tuple[str, int, int]
+
+
+def self_write_key(path: str) -> str:
+    """Normalise a path to the form both services store and look up by."""
+    return os.path.normcase(os.path.abspath(path))
+
+
+def _connect() -> sqlite3.Connection:
+    conn = sqlite3.connect(DATABASE_PATH)
+    # This table stands alone, keyed by path, so nothing here depends on it
+    # today. Set anyway to match every other _connect() in this package.
+    conn.execute("PRAGMA foreign_keys = ON")
+    return conn
+
+
+def db_create_self_writes_table() -> None:
+    conn = _connect()
+    cursor = conn.cursor()
+    try:
+        cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS self_writes (
+                path TEXT PRIMARY KEY,
+                file_size INTEGER NOT NULL,
+                file_mtime INTEGER NOT NULL,
+                -- Epoch seconds rather than CURRENT_TIMESTAMP: this column only
+                -- exists to be subtracted from, and text timestamps make that
+                -- arithmetic a timezone question.
+                written_at INTEGER NOT NULL
+            )
+        """
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def db_record_self_write(path: str, file_size: int, file_mtime: int) -> bool:
+    """
+    Note that PictoPy is about to leave a file in this exact state.
+
+    Callers record before the bytes land, because the watcher can fire the
+    instant they do.
+    """
+    conn = _connect()
+    cursor = conn.cursor()
+    now = int(time.time())
+
+    try:
+        cursor.execute(
+            """
+            INSERT INTO self_writes (path, file_size, file_mtime, written_at)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(path) DO UPDATE SET
+                file_size=excluded.file_size,
+                file_mtime=excluded.file_mtime,
+                written_at=excluded.written_at
+            """,
+            (self_write_key(path), file_size, file_mtime, now),
+        )
+        # Pruning here avoids needing a scheduler for a table this small.
+        cursor.execute(
+            "DELETE FROM self_writes WHERE written_at < ?",
+            (now - SELF_WRITE_TTL_SECONDS,),
+        )
+        conn.commit()
+        return True
+    except sqlite3.Error as e:
+        logger.error(f"Error recording self write for {path}: {e}")
+        conn.rollback()
+        return False
+    finally:
+        conn.close()
+
+
+def db_take_matching_self_writes(observed: List[ObservedFile]) -> Set[str]:
+    """
+    Return the observed paths that match a recorded write, and forget them.
+
+    Claiming an entry as it matches means a second event for the same write is
+    treated as a real change. That is the safe direction to be wrong in: it
+    costs one redundant rescan, where holding the entry could swallow a genuine
+    edit that happened to land in the same second at the same size.
+    """
+    if not observed:
+        return set()
+
+    conn = _connect()
+    cursor = conn.cursor()
+    matched: Set[str] = set()
+
+    try:
+        by_key = {self_write_key(path): path for path, _, _ in observed}
+        placeholders = ",".join("?" for _ in observed)
+        cursor.execute(
+            f"""
+            SELECT path, file_size, file_mtime
+            FROM self_writes
+            WHERE path IN ({placeholders})
+            """,
+            list(by_key),
+        )
+        recorded = {row[0]: (row[1], row[2]) for row in cursor.fetchall()}
+
+        claimed = []
+        for path, size, mtime in observed:
+            key = self_write_key(path)
+            if recorded.get(key) == (size, mtime):
+                matched.add(path)
+                claimed.append(key)
+
+        if claimed:
+            cursor.executemany(
+                "DELETE FROM self_writes WHERE path = ?",
+                [(key,) for key in claimed],
+            )
+            conn.commit()
+
+        return matched
+    except sqlite3.Error as e:
+        # Failing open means a redundant rescan, never a missed change.
+        logger.error(f"Error matching self writes: {e}")
+        return set()
+    finally:
+        conn.close()

backend/app/utils/self_write.py

@@ -0,0 +1,70 @@
+"""
+Replacing a file in a watched folder without the change reading as a user edit.
+
+Anything that rewrites a file inside a registered folder should go through here
+rather than opening the path directly.
+"""
+
+import os
+import tempfile
+from typing import Optional
+
+from app.database.self_writes import db_record_self_write
+from app.logging.setup_logging import get_logger
+
+logger = get_logger(__name__)
+
+# The temp file lands in the watched folder too, so the watcher recognises and
+# ignores it by name. Mirrored in sync-microservice/app/utils/watcher.py.
+SELF_WRITE_TEMP_PREFIX = ".pictopy-write-"
+
+
+def _discard(temp_path: Optional[str]) -> None:
+    """Best-effort cleanup; a leftover temp is noise, not a failure worth raising."""
+    if not temp_path:
+        return
+    try:
+        os.unlink(temp_path)
+    except OSError:
+        logger.warning(f"Could not remove temporary file {temp_path}")
+
+
+def self_write_util_replace(path: str, data: bytes) -> bool:
+    """
+    Atomically replace a file's contents and record the write for the watcher.
+
+    Writes to a sibling temp file, records the size and mtime that file already
+    has, then renames it into place. The rename carries both across unchanged,
+    so the ledger row is in the database before the new bytes are visible at the
+    watched path -- which matters, because the watcher can fire the moment they
+    are.
+
+    Returns False if the file could not be replaced, leaving the original as is.
+    """
+    target = os.path.abspath(path)
+    directory = os.path.dirname(target)
+    temp_path: Optional[str] = None
+
+    try:
+        # Same directory, so the rename stays on one filesystem and stays atomic.
+        handle_fd, temp_path = tempfile.mkstemp(
+            dir=directory, prefix=SELF_WRITE_TEMP_PREFIX, suffix=".tmp"
+        )
+        with os.fdopen(handle_fd, "wb") as handle:
+            handle.write(data)
+            handle.flush()
+            # Without this the rename can land before the bytes do, which on a
+            # crash leaves a photo that is neither the old one nor the new one.
+            os.fsync(handle.fileno())
+
+        stats = os.stat(temp_path)
+        db_record_self_write(target, stats.st_size, int(stats.st_mtime))
+
+        os.replace(temp_path, target)
+        return True
+    except OSError as e:
+        # A locked or read-only file is ordinary on Windows; the caller retries
+        # on a later pass rather than treating it as a failed run.
+        logger.error(f"Could not replace {path}: {e}")
+        _discard(temp_path)
+        return False

backend/main.py

@@ -25,6 +25,7 @@
     db_clear_stale_processing_flags,
 )
 from app.database.metadata import db_create_metadata_table
+from app.database.self_writes import db_create_self_writes_table
 from app.database.semantic_labels import db_create_semantic_labels_table
 from app.database.image_embeddings import db_create_image_embeddings_table
 from app.database.video_frames import db_create_video_frames_tables
@@ -83,6 +84,7 @@ async def lifespan(app: FastAPI):
     db_create_album_images_table()
     db_create_metadata_table()
     db_create_memories_table()  # References images(id) and videos(id)
+    db_create_self_writes_table()  # Standalone: keyed by path, no foreign keys
     # Nothing is indexing or tagging yet, so anything still flagged busy is
     # left over from a previous session and would block memory generation.
     db_clear_stale_processing_flags()

backend/tests/conftest.py

@@ -10,6 +10,7 @@
 from app.database.albums import db_create_albums_table, db_create_album_images_table
 from app.database.folders import db_create_folders_table
 from app.database.metadata import db_create_metadata_table
+from app.database.self_writes import db_create_self_writes_table
 from app.database.semantic_labels import db_create_semantic_labels_table
 from app.database.image_embeddings import db_create_image_embeddings_table
 from app.database.video_frames import db_create_video_frames_tables
@@ -39,6 +40,7 @@ def setup_before_all_tests():
         db_create_video_frames_tables()
         db_create_metadata_table()
         db_create_memories_table()  # References images(id) and videos(id)
+        db_create_self_writes_table()  # Standalone: keyed by path, no foreign keys
         print("All database tables created successfully")
     except Exception as e:
         print(f"Error creating database tables: {e}")