diff --git a/docs/admin-guide.md b/docs/admin-guide.md index b93cf697..e61f69e7 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -693,13 +693,17 @@ Run `stack up backup`. You need an APFS-formatted external drive plugged in. The | Source | Path on the archive disk | |---|---| -| Immich photo originals | `/Volumes//data/photos-library/` | -| Paperless archived PDFs | `/Volumes//data/docs-media/` | -| Matrix uploads: voice messages, photos, files | `/Volumes//data/messages-media/` | -| Matrix timeline, as dated snapshots | `/Volumes//data/messages-synapse/` | +| Immich photo originals | `/Volumes//data/photos/library/` | +| Paperless archived PDFs | `/Volumes//data/docs/media/` | +| Matrix uploads: voice messages, photos, files | `/Volumes//data/messages/media/` | +| Matrix timeline, as dated snapshots | `/Volumes//data/messages/synapse/` | Immich's and Paperless's Postgres databases are still not covered. You get those files back but lose albums, tags, custom fields and saved views. They use the same snapshot mechanism the chat server already uses, so wiring them up is a small change rather than a new design. +**Disks from an earlier release** + +Backups written by 0.3.0-beta.3 and earlier used one flat directory per source, `data/photos-library/` where this release writes `data/photos/library/`. Both layouts are read, and a sync keeps using whichever directory it finds, so leaving it alone is a valid choice. To move to the current layout, connect the disk and run `stack backup migrate`. It renames the directories in place, so nothing is copied however much is on the disk, and the files keep the immutable flag that makes the archive append-only. Add `--dry-run` to see what it would do first. + **How the protection works** Every file written to the archive gets the kernel `uchg` flag. macOS refuses to modify or delete uchg files, even with `sudo`. `rsync --ignore-existing` means files already in the archive are skipped on every run, so the backup is append-only by design and accidental `rm -rf` on your main system cannot propagate. @@ -737,8 +741,8 @@ Recover the snapshot and the media together, and do the database first. Media th ```bash # 1. Take the snapshot off the archive disk and unpack it -sudo chflags nouchg /Volumes//data/messages-synapse/synapse-.tar.gz -cp /Volumes//data/messages-synapse/synapse-.tar.gz ~/ +sudo chflags nouchg /Volumes//data/messages/synapse/synapse-.tar.gz +cp /Volumes//data/messages/synapse/synapse-.tar.gz ~/ tar xzf ~/synapse-.tar.gz -C ~/restore/ # 2. Stop the homeserver so nothing writes while you work @@ -755,8 +759,8 @@ cp ~/restore/homeserver.yaml ~/restore/*.signing.key \ ~/famstack-data/messages/synapse/ # 5. Put the recordings back -sudo chflags -R nouchg /Volumes//data/messages-media/ -cp -R /Volumes//data/messages-media/* \ +sudo chflags -R nouchg /Volumes//data/messages/media/ +cp -R /Volumes//data/messages/media/* \ ~/famstack-data/messages/synapse/media_store/local_content/ ./stack up messages @@ -780,8 +784,8 @@ The scheduled nightly run leaves the disk mounted between runs (cron cannot trig Plug the archive disk into any Mac and browse the files in Finder. To copy locked files out: ```bash -sudo chflags -R nouchg /Volumes//data/photos-library// -cp -R /Volumes//data/photos-library// ~/recovered/ +sudo chflags -R nouchg /Volumes//data/photos/library// +cp -R /Volumes//data/photos/library// ~/recovered/ ``` A `stack backup restore` command and `on_restore` hooks for database recovery are planned but not yet shipped. diff --git a/docs/stack-reference.md b/docs/stack-reference.md index 00fa3b0f..6e1a00ca 100644 --- a/docs/stack-reference.md +++ b/docs/stack-reference.md @@ -381,7 +381,7 @@ path = "{data_dir}/photos/library/library" | Field | Description | |---|---| -| `name` | Short slug for this source. Combined with the stacklet id, this becomes the global source id (`photos/library`). Used in `stack backup status` output and (future) `--source=` selection. | +| `name` | Short slug for this source. Combined with the stacklet id, this becomes the global source id (`photos/library`), which is also the directory it occupies on a vault (`data/photos/library/`). Used in `stack backup status` output and (future) `--source=` selection. | | `path` | Filesystem path to sync. Template variables from the rendered environment are available (`{data_dir}`, etc.). | There is no threshold to declare. The engine judges each source against diff --git a/stacklets/backup/README.md b/stacklets/backup/README.md index 3e0e6542..5f02b8c6 100644 --- a/stacklets/backup/README.md +++ b/stacklets/backup/README.md @@ -89,6 +89,7 @@ engine supports append-only semantics. Adding a second target later ``` stack backup sync [--dry-run] [--no-eject] [--verbose] stack backup status # last run, source counts, cron presence +stack backup migrate [--dry-run] # move a vault to the current layout ``` Per-stacklet aliases (`stack photos backup`, `stack docs backup`) and @@ -96,6 +97,22 @@ restore (`stack backup restore --source=…`) are intentionally not in v1 — they'll layer on once the engine port lands and the manifest contract has been exercised on at least one production sync. +## Vault layout + +Each source owns one directory under `data/`, named after its id: +`photos/library` lands in `data/photos/library/`. One directory per +stacklet keeps everything that has to be restored together in one +place, and it stays unambiguous when a stacklet id or a source name +contains a hyphen. + +Vaults written by 0.3.0-beta.3 and earlier hold the flat form, +`data/photos-library/`. The engine reads both and keeps writing into +whichever directory it finds, because adopting the new path would copy +every file a second time and the old tree could not be removed +afterwards: its files are locked immutable. `stack backup migrate` +renames the directories in place, which moves no data and preserves the +locks. + ## Guarding the sources Two checks run before anything is written. diff --git a/stacklets/backup/cli/_orchestrator.py b/stacklets/backup/cli/_orchestrator.py index c86097ae..c65db9f0 100644 --- a/stacklets/backup/cli/_orchestrator.py +++ b/stacklets/backup/cli/_orchestrator.py @@ -66,6 +66,29 @@ class Target: schedule: str # Cron expression; informational at this level +# ── Vault layout ─────────────────────────────────────────────────────────── + +def vault_subdir(source_id: str) -> str: + """Where a source's files live on the vault, relative to the mount. + + The source id is already ``{stacklet_id}/{name}``, so the vault + mirrors it: ``data/messages/synapse``. One directory per stacklet + groups everything that has to be restored together. + """ + return f"data/{source_id}" + + +def legacy_vault_subdir(source_id: str) -> str: + """The flat form used by vaults written with 0.3.0-beta.3 or earlier. + + Both halves of a source id may contain hyphens, so ``data/{id with + the slash replaced}`` cannot be split back into stacklet and name. + It is still read, because those directories exist on disks in the + field; ``stack backup migrate`` renames them. + """ + return f"data/{source_id.replace('/', '-')}" + + # ── Source discovery ─────────────────────────────────────────────────────── def discover_archive_sources( @@ -81,10 +104,8 @@ def discover_archive_sources( (currently just ``{data_dir}``) are rendered into the path field. The source ``id`` is ``{stacklet_id}/{archive.name}`` so a single - stacklet can declare multiple archives without collision. The - vault subdirectory is derived as ``data/{stacklet_id}-{name}`` — - short, stable, and namespaced so future stacklets can't accidentally - clobber existing archive directories. + stacklet can declare multiple archives without collision, and the + vault subdirectory mirrors it (see :func:`vault_subdir`). """ stacklets_dir = repo_root / "stacklets" if not stacklets_dir.is_dir(): @@ -119,11 +140,12 @@ def discover_archive_sources( # will surface the problem with a useful error. rendered_path = raw_path + source_id = f"{stacklet_id}/{name}" sources.append(SourceRecord( - id=f"{stacklet_id}/{name}", + id=source_id, display=stacklet_display, src_path=Path(rendered_path), - vault_subdir=f"data/{stacklet_id}-{name}", + vault_subdir=vault_subdir(source_id), )) return sources diff --git a/stacklets/backup/cli/_snapshot.py b/stacklets/backup/cli/_snapshot.py index d1ba294b..a823a970 100644 --- a/stacklets/backup/cli/_snapshot.py +++ b/stacklets/backup/cli/_snapshot.py @@ -38,7 +38,7 @@ from stack import postgres -from _orchestrator import SourceRecord +from _orchestrator import SourceRecord, vault_subdir # Tarballs retained on the internal disk. The vault retains all of them, @@ -50,7 +50,10 @@ class SnapshotSpec: """One ``[[backup.snapshot]]`` entry, after template rendering.""" - id: str # "{stacklet_id}/{name}", e.g. "messages/synapse" + # "{stacklet_id}/{name}", e.g. "messages/synapse". Doubles as the + # directory holding this snapshot's tarballs, both in the staging + # area and under the vault's `data/`. + id: str display: str # Human-readable, e.g. "Messages" name: str # "synapse" # Capture parameters, keyed by the mechanism that reads them. The @@ -75,13 +78,6 @@ def user(self) -> str: def stacklet_id(self) -> str: return self.id.split("/", 1)[0] - @property - def subdir(self) -> str: - """Directory holding this snapshot's tarballs, used both on the - internal disk and under the vault's `data/`. Qualified by stacklet - so two stacklets choosing the same `name` do not collide.""" - return f"{self.stacklet_id}-{self.name}" - # ── Discovery ────────────────────────────────────────────────────────────── @@ -229,7 +225,7 @@ def take_snapshot( # Taken first, so a failure propagates before any file exists. sql = dump(spec) - out_dir = out_root / spec.subdir + out_dir = out_root / spec.id out_dir.mkdir(parents=True, exist_ok=True) # The timestamp has second resolution, so two runs within the same # second would otherwise produce the same name. A suffix keeps them @@ -334,8 +330,8 @@ def snapshot_source(spec: SnapshotSpec, out_root: Path) -> SourceRecord: return SourceRecord( id=spec.id, display=spec.display, - src_path=out_root / spec.subdir, - vault_subdir=f"data/{spec.subdir}", + src_path=out_root / spec.id, + vault_subdir=vault_subdir(spec.id), rolling=True, ) diff --git a/stacklets/backup/cli/migrate.py b/stacklets/backup/cli/migrate.py new file mode 100644 index 00000000..17eefcd6 --- /dev/null +++ b/stacklets/backup/cli/migrate.py @@ -0,0 +1,182 @@ +"""stack backup migrate — move a vault to the current directory layout. + +Vaults written with 0.3.0-beta.3 or earlier hold one flat directory per +source: ``data/messages-synapse``. Current releases nest them under the +stacklet, ``data/messages/synapse``, so everything belonging to one +stacklet restores from one place and a hyphen inside a stacklet id or a +source name stays unambiguous. + +Both layouts are read. A sync keeps using whichever flat directory it +finds, so an un-migrated vault goes on working and no file is ever +copied twice. This command performs the one-time rename. + +What moves is the directory, not its contents: the files inside keep the +immutable flag that makes the vault append-only, and nothing is copied. +The disk must already be mounted, because unlocking and mounting it is +``stack backup sync``'s job. + +Usage: + stack backup migrate rename legacy directories on every target + stack backup migrate --dry-run list what would be renamed +""" + +HELP = "Move a vault's data directories to the current layout" + +import argparse +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import List, Tuple + +_here = Path(__file__).parent +sys.path.insert(0, str(_here)) +from _orchestrator import ( # noqa: E402 + discover_archive_sources, + get_targets, + legacy_vault_subdir, + vault_subdir, +) +from _snapshot import discover_snapshots # noqa: E402 + + +@dataclass +class Move: + """One source's directory on one vault, and what became of it.""" + + display: str + old: Path + new: Path + status: str # "moved" | "occupied" | "failed" + reason: str = "" + + +# ── Planning ─────────────────────────────────────────────────────────────── + +def declared_sources( + repo_root: Path, instance_dir: Path, data_dir: Path, +) -> List[Tuple[str, str]]: + """Every source that could own a directory on a vault, as (id, display). + + Archives and snapshots both land in ``data/`` under their source id, + so both are candidates. A stacklet that has since been disabled is + not discovered and keeps its flat directory, which stays readable. + """ + found = { + s.id: s.display + for s in discover_archive_sources(repo_root, instance_dir, data_dir) + } + for spec in discover_snapshots(repo_root, instance_dir, data_dir): + found.setdefault(spec.id, spec.display) + return sorted(found.items()) + + +def migrate_vault( + mount_point: Path, sources: List[Tuple[str, str]], *, dry_run: bool, +) -> List[Move]: + """Rename each legacy directory found on this vault, and report. + + A source whose new directory already exists is left alone. That + means both layouts hold data, which no rename can reconcile: the + files are immutable, so they cannot be merged into one tree without + unlocking them, and choosing one would hide the other. + """ + moves: List[Move] = [] + + for source_id, display in sources: + old = mount_point / legacy_vault_subdir(source_id) + new = mount_point / vault_subdir(source_id) + if not old.is_dir(): + continue + + if new.exists(): + moves.append(Move(display, old, new, "occupied")) + continue + + if not dry_run: + try: + new.parent.mkdir(parents=True, exist_ok=True) + old.rename(new) + except OSError as e: + moves.append(Move(display, old, new, "failed", str(e))) + continue + + moves.append(Move(display, old, new, "moved")) + + return moves + + +# ── Output ───────────────────────────────────────────────────────────────── + +def _render(target_name: str, moves: List[Move], dry_run: bool) -> None: + from stack.prompt import bold, dim, done, error, nl, warn + + nl() + bold(f"Target '{target_name}'") + if not moves: + dim("Already on the current layout.") + return + + for m in moves: + nested = f"{m.new.parent.name}/{m.new.name}" + if m.status == "moved": + verb = "would move" if dry_run else "moved" + done(f"{m.display}: {verb} {m.old.name} to {nested}") + elif m.status == "occupied": + warn(f"{m.display}: both {m.old.name} and {nested} hold data. " + "Merge them by hand, then re-run.") + else: + error(f"{m.display}: could not move {m.old.name} ({m.reason})") + + +# ── Entry point ──────────────────────────────────────────────────────────── + +def _parse_args(argv: list) -> argparse.Namespace: + parser = argparse.ArgumentParser(prog="stack backup migrate", + description=HELP) + parser.add_argument("--dry-run", action="store_true", + help="List what would be renamed (no changes).") + return parser.parse_args(argv) + + +def run(args, stacklet, config): + """Entry point invoked by the framework via ``stack backup migrate``.""" + parsed = _parse_args(args or []) + + repo_root = Path(config.get("repo_root", ".")) + instance_dir = Path(config.get("instance_dir", repo_root)) + data_dir = Path(config.get("data_dir", ".")) + + targets = get_targets(config.get("stack", {})) + if not targets: + return { + "error": "No backup targets configured. Add a [backup.targets.] " + "block to stack.toml (see stack.example.toml)." + } + + sources = declared_sources(repo_root, instance_dir, data_dir) + unreachable: List[str] = [] + problems = 0 + migrated = 0 + + for target in targets: + mount_point = Path("/Volumes") / target.disk + if not mount_point.is_dir(): + unreachable.append(target.name) + continue + + moves = migrate_vault(mount_point, sources, dry_run=parsed.dry_run) + _render(target.name, moves, parsed.dry_run) + migrated += sum(1 for m in moves if m.status == "moved") + problems += sum(1 for m in moves if m.status != "moved") + + if unreachable and len(unreachable) == len(targets): + return { + "error": "No backup disk is mounted. Connect it and run " + "'stack backup sync' once, which unlocks and mounts it." + } + for name in unreachable: + print(f" [{name}] disk not mounted, skipped", file=sys.stderr) + + if problems: + return {"error": "Some directories could not be moved; see above"} + return {"ok": True, "moved": migrated, "dry_run": parsed.dry_run} diff --git a/stacklets/backup/cli/sync.py b/stacklets/backup/cli/sync.py index 40999e24..3277beef 100644 --- a/stacklets/backup/cli/sync.py +++ b/stacklets/backup/cli/sync.py @@ -92,7 +92,7 @@ def _take_snapshots( continue size_kb = max(1, path.stat().st_size // 1024) print(f" {spec.display}: {path.name} ({size_kb} KB)") - prune_snapshots(out_root / spec.subdir) + prune_snapshots(out_root / spec.id) sources.append(snapshot_source(spec, out_root)) return sources, failed diff --git a/stacklets/backup/engines/external-disk/README.md b/stacklets/backup/engines/external-disk/README.md index ef71eeef..2763d853 100644 --- a/stacklets/backup/engines/external-disk/README.md +++ b/stacklets/backup/engines/external-disk/README.md @@ -87,7 +87,7 @@ call, no separate process needed. |---|---| | `BACKUP_DATA_DIR` | Host-side state directory (canary, logs, result JSON). Refused if under `/Volumes/`. | | `VAULT_DISK` | APFS volume name. Mount point is `/Volumes/`. | -| `SOURCES` | Newline-separated, pipe-delimited records: `\|\|\|\|` | +| `SOURCES` | Newline-separated, pipe-delimited records: `\|\|\|\|` | Arguments are POSIX-style: `--dry-run`, `--no-eject`, `--verbose`, `--verify`. diff --git a/stacklets/backup/engines/external-disk/sync.py b/stacklets/backup/engines/external-disk/sync.py index 9f26f1b3..e3502f9f 100755 --- a/stacklets/backup/engines/external-disk/sync.py +++ b/stacklets/backup/engines/external-disk/sync.py @@ -240,6 +240,40 @@ def parse_sources(sources_env: str) -> List[Source]: return records +# ── Vault layout ─────────────────────────────────────────────────────────── + +def legacy_vault_subdir(vault_subdir: str) -> str: + """The flat directory name a vault used before the nested layout. + + ``data/messages/synapse`` was written as ``data/messages-synapse``. + The two encodings of the same source id differ only in the + separator before the last component. + """ + head, _, name = vault_subdir.rpartition("/") + return f"{head}-{name}" + + +def vault_dir(mount_point: Path, src: Source) -> Path: + """The directory on this vault holding the source's files. + + A vault written by an earlier release keeps its flat directory until + ``stack backup migrate`` renames it. Switching without that rename + would copy every file again under the new path, and the flat tree + could not be removed afterwards: its files are locked immutable. + + An empty flat directory does not count. It is what a migration + leaves behind if the rename is interrupted, and following it would + strand the data that did move. + """ + dest = mount_point / src.vault_subdir + if dest.is_dir(): + return dest + legacy = mount_point / legacy_vault_subdir(src.vault_subdir) + if legacy.is_dir() and any(legacy.iterdir()): + return legacy + return dest + + # ── Number formatting ────────────────────────────────────────────────────── def format_number(n: int) -> str: @@ -719,7 +753,12 @@ def sync_data( results: List[SourceResult] = [] for src in sources: - dest = mount_point / src.vault_subdir + dest = vault_dir(mount_point, src) + if dest != mount_point / src.vault_subdir: + warn( + f"{src.display}: vault uses the old flat layout " + f"({dest.name}/). Run 'stack backup migrate' to update it." + ) before_count = count_files(dest) if dest.is_dir() else 0 if not dry_run: @@ -836,7 +875,7 @@ def verify_sync(sources: List[Source], mount_point: Path) -> None: header("Verifying sync") for src in sources: - dest = mount_point / src.vault_subdir + dest = vault_dir(mount_point, src) src_count = count_files(src.src_path) dest_count = count_files(dest) if dest_count >= src_count: diff --git a/stacklets/backup/stacklet.toml b/stacklets/backup/stacklet.toml index 4e4becb2..570bc390 100644 --- a/stacklets/backup/stacklet.toml +++ b/stacklets/backup/stacklet.toml @@ -5,7 +5,8 @@ # # 1. Files locked with chflags uchg (kernel refuses modify or delete). # 2. rsync --ignore-existing (append-only: never overwrites existing). -# 3. Canary + min-file-count preflight (ransomware tripwire). +# 3. Canary + a preflight that measures each source against what +# it held on the previous run (ransomware tripwire). # 4. APFS encryption optional (defends physical theft only). # 5. Eject after sync — works from Terminal, sandbox-blocked from cron. # Treat as a bonus, not a guarantee: the disk usually stays mounted diff --git a/tests/stacklets/test_backup_e2e.py b/tests/stacklets/test_backup_e2e.py index fb3b84b1..5a0bf53c 100644 --- a/tests/stacklets/test_backup_e2e.py +++ b/tests/stacklets/test_backup_e2e.py @@ -38,6 +38,10 @@ reason="backup E2E tests require macOS (hdiutil + chflags + APFS)", ) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / + "stacklets" / "backup" / "cli")) +from migrate import migrate_vault # noqa: E402 + REPO_ROOT = Path(__file__).resolve().parents[2] ENGINE_SCRIPT = REPO_ROOT / "stacklets" / "backup" / "engines" / "external-disk" / "sync.py" STACK_BIN = REPO_ROOT / "stack" @@ -140,8 +144,8 @@ def _sources_env(fake_sources: dict, *, rolling: bool = False) -> str: """ flag = 1 if rolling else 0 return "\n".join([ - f"photos/library|Photos|{fake_sources['photos']}|data/photos-library|{flag}", - f"docs/media|Documents|{fake_sources['docs']}|data/docs-media|{flag}", + f"photos/library|Photos|{fake_sources['photos']}|data/photos/library|{flag}", + f"docs/media|Documents|{fake_sources['docs']}|data/docs/media|{flag}", ]) @@ -214,8 +218,8 @@ def test_first_sync_writes_and_locks_files( f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" ) - vault_photos = mount / "data" / "photos-library" - vault_docs = mount / "data" / "docs-media" + vault_photos = mount / "data" / "photos" / "library" + vault_docs = mount / "data" / "docs" / "media" assert vault_photos.is_dir() assert vault_docs.is_dir() @@ -244,7 +248,7 @@ def test_immutable_files_resist_modification( _run_engine(backup_data_dir, name, _sources_env(fake_sources), args=["--no-eject"]) - locked = next((mount / "data" / "photos-library").glob("*.jpg")) + locked = next((mount / "data" / "photos" / "library").glob("*.jpg")) with pytest.raises(PermissionError): locked.write_bytes(b"tampered") @@ -257,7 +261,7 @@ def test_second_sync_is_noop( sources = _sources_env(fake_sources) _run_engine(backup_data_dir, name, sources, args=["--no-eject"]) - photos = mount / "data" / "photos-library" + photos = mount / "data" / "photos" / "library" before = {p.name: p.stat().st_mtime for p in photos.iterdir()} result = _run_engine(backup_data_dir, name, sources, args=["--no-eject"]) @@ -283,7 +287,7 @@ def test_new_source_files_picked_up_on_next_run( result = _run_engine(backup_data_dir, name, sources, args=["--no-eject"]) assert result.returncode == 0 - new_on_vault = mount / "data" / "photos-library" / "extra.jpg" + new_on_vault = mount / "data" / "photos" / "library" / "extra.jpg" assert new_on_vault.is_file() assert _has_uchg(new_on_vault) @@ -356,7 +360,7 @@ def test_refuses_when_a_source_lost_files_since_last_run( # The vault stays clean — preflight failure means we never # mounted (or in this case never wrote, since the disk was # already mounted by the test fixture). - assert not (mount / "data" / "photos-library").exists() + assert not (mount / "data" / "photos" / "library").exists() data = _read_result(backup_data_dir) assert data["success"] is False @@ -507,8 +511,8 @@ def test_stack_backup_sync_runs_engine_end_to_end( f"stderr:\n{result.stderr}" ) - vault_photos = mount / "data" / "photos-library" - vault_docs = mount / "data" / "docs-media" + vault_photos = mount / "data" / "photos" / "library" + vault_docs = mount / "data" / "docs" / "media" assert vault_photos.is_dir(), \ f"orchestrator didn't write photos to vault.\nstdout:\n{result.stdout}" assert vault_docs.is_dir() @@ -527,11 +531,14 @@ class TestUpgradingAnExistingVault: """An instance that has been backing up photos and documents for months, upgraded to a release that adds snapshots and a new archive. - Two things could go wrong. The shrink check reads each source's count - from the previous run, and every run recorded before the upgrade - predates that field, so an absent baseline must not read as loss. And - the new sources must be added to the vault without disturbing what is - already on it, which is immutable and cannot be rewritten. + Three things could go wrong. The shrink check reads each source's + count from the previous run, and every run recorded before the + upgrade predates that field, so an absent baseline must not read as + loss. The new sources must be added to the vault without disturbing + what is already on it, which is immutable and cannot be rewritten. + And the directories already on the vault use the flat layout that + release wrote, which this one has to go on using until the household + runs `stack backup migrate`. """ def _existing_vault(self, mount: Path) -> dict: @@ -579,14 +586,14 @@ def test_the_first_upgraded_sync_adds_without_disturbing( media.mkdir(parents=True) for i in range(3): (media / f"voice-{i}.ogg").write_text("audio") - snaps = tmp_path / "data" / "snapshots" / "messages-synapse" + snaps = tmp_path / "data" / "snapshots" / "messages" / "synapse" snaps.mkdir(parents=True) (snaps / "synapse-20260911T000000Z.tar.gz").write_text("dump") sources = "\n".join([ _sources_env(fake_sources), - f"messages/media|Messages|{media}|data/messages-media|0", - f"messages/synapse|Messages|{snaps}|data/messages-synapse|1", + f"messages/media|Messages|{media}|data/messages/media|0", + f"messages/synapse|Messages|{snaps}|data/messages/synapse|1", ]) result = _run_engine(backup_data_dir, name, sources, args=["--no-eject"]) assert result.returncode == 0 @@ -600,9 +607,15 @@ def test_the_first_upgraded_sync_adds_without_disturbing( now = {p.name for p in (mount / "data" / subdir).iterdir()} assert set(names) <= now - # And the new sources arrived. - assert len(list((mount / "data" / "messages-media").iterdir())) == 3 - assert len(list((mount / "data" / "messages-synapse").iterdir())) == 1 + # The photos went into the directory that was already there, + # rather than starting a second copy under the new layout. + assert len(list((mount / "data" / "photos-library").glob("*.jpg"))) == 15 + assert not (mount / "data" / "photos").exists() + + # New sources have no directory to inherit, so they get the + # current layout. + assert len(list((mount / "data" / "messages" / "media").iterdir())) == 3 + assert len(list((mount / "data" / "messages" / "synapse").iterdir())) == 1 def test_it_records_a_baseline_the_next_run_can_use( self, vault_image, backup_data_dir, fake_sources, @@ -624,3 +637,57 @@ def test_it_records_a_baseline_the_next_run_can_use( # rule rather than a number that drifts with the fixture. assert counts["photos/library"] == _count_files(fake_sources["photos"]) assert counts["docs/media"] == _count_files(fake_sources["docs"]) + + +class TestMigratingAVault: + """`stack backup migrate`, against a vault holding real immutable files. + + The rename has to leave the vault in a state the engine recognises: + it goes on adding to the moved directory, and the files already + there are neither re-copied nor unlocked. + """ + + def _legacy_env(self, fake_sources: dict) -> str: + """What the engine was told before the layout changed.""" + return "\n".join([ + f"photos/library|Photos|{fake_sources['photos']}|data/photos-library|0", + f"docs/media|Documents|{fake_sources['docs']}|data/docs-media|0", + ]) + + def test_a_migrated_vault_keeps_syncing_incrementally( + self, vault_image, backup_data_dir, fake_sources, + ): + name, mount = vault_image + + # An earlier release fills the vault with flat directories. + first = _run_engine(backup_data_dir, name, + self._legacy_env(fake_sources), args=["--no-eject"]) + assert first.returncode == 0 + assert len(list((mount / "data" / "photos-library").glob("*.jpg"))) == 15 + + moves = migrate_vault( + mount, [("docs/media", "Documents"), ("photos/library", "Photos")], + dry_run=False, + ) + assert [m.status for m in moves] == ["moved", "moved"] + + photos = mount / "data" / "photos" / "library" + assert len(list(photos.glob("*.jpg"))) == 15 + assert not (mount / "data" / "photos-library").exists() + # The move carried the immutability with it. A copy could not: + # the lock is applied to new files after a sync, and an unlocked + # window is what the vault design exists to exclude. + assert _has_uchg(next(photos.glob("*.jpg"))) + + # One new photo since. The next sync adds that and nothing else, + # which is only true if it recognised the moved directory. + (Path(fake_sources["photos"]) / "photo-015.jpg").write_bytes(b"x" * 256) + second = _run_engine(backup_data_dir, name, _sources_env(fake_sources), + args=["--no-eject"]) + assert second.returncode == 0 + + result = next(s for s in _read_result(backup_data_dir)["sources"] + if s["id"] == "photos/library") + assert result["new_files"] == 1 + assert result["total_files"] == 16 + assert not (mount / "data" / "photos-library").exists() diff --git a/tests/stacklets/test_backup_engine.py b/tests/stacklets/test_backup_engine.py index 3bb553c6..26e5bde7 100644 --- a/tests/stacklets/test_backup_engine.py +++ b/tests/stacklets/test_backup_engine.py @@ -33,6 +33,7 @@ previous_source_counts, probe_filesystem, read_latest_run, + vault_dir, verify_canary, ) @@ -42,14 +43,14 @@ class TestParseSources: def test_single_record(self): sources = parse_sources( - "photos/library|Photos|/data/photos/library|data/photos-library|0" + "photos/library|Photos|/data/photos/library|data/photos/library|0" ) assert len(sources) == 1 s = sources[0] assert s.id == "photos/library" assert s.display == "Photos" assert s.src_path == Path("/data/photos/library") - assert s.vault_subdir == "data/photos-library" + assert s.vault_subdir == "data/photos/library" assert s.rolling is False def test_multiple_records_separated_by_newlines(self): @@ -90,6 +91,54 @@ def test_too_many_fields_aborts(self): parse_sources("a|b|c|d|0|extra") +# ── Vault layout ─────────────────────────────────────────────────────────── + +class TestVaultDir: + """Where a source's files go on the vault. + + The current layout nests a source under its stacklet, + `data/photos/library`. Vaults written by 0.3.0-beta.3 and earlier + hold `data/photos-library`, and those directories stay in use until + the household runs `stack backup migrate`: the files in them are + locked immutable, so adopting the new path would copy every one + again and leave the old tree undeletable. + """ + + def _source(self) -> Source: + return Source(id="photos/library", display="Photos", + src_path=Path("/src"), vault_subdir="data/photos/library") + + def test_a_fresh_vault_gets_the_nested_layout(self, tmp_path): + assert vault_dir(tmp_path, self._source()) == \ + tmp_path / "data" / "photos" / "library" + + def test_an_existing_flat_directory_keeps_being_used(self, tmp_path): + flat = tmp_path / "data" / "photos-library" + flat.mkdir(parents=True) + (flat / "holiday.jpg").write_text("x") + + assert vault_dir(tmp_path, self._source()) == flat + + def test_an_empty_flat_directory_does_not_count(self, tmp_path): + """What an interrupted migration leaves behind. Following it + would strand the files that did move.""" + (tmp_path / "data" / "photos-library").mkdir(parents=True) + + assert vault_dir(tmp_path, self._source()) == \ + tmp_path / "data" / "photos" / "library" + + def test_a_migrated_vault_never_looks_back(self, tmp_path): + """Both directories present means the migration ran and left an + empty husk, or someone made one by hand. The nested one wins.""" + (tmp_path / "data" / "photos" / "library").mkdir(parents=True) + flat = tmp_path / "data" / "photos-library" + flat.mkdir(parents=True) + (flat / "holiday.jpg").write_text("x") + + assert vault_dir(tmp_path, self._source()) == \ + tmp_path / "data" / "photos" / "library" + + # ── Canary ───────────────────────────────────────────────────────────────── class TestVerifyCanary: @@ -130,7 +179,7 @@ def _make_source(self, tmp_path: Path, name: str, file_count: int) -> Source: id=f"test/{name}", display=name.title(), src_path=src_dir, - vault_subdir=f"data/test-{name}", + vault_subdir=f"data/test/{name}", ) def test_a_first_ever_run_syncs_whatever_is_there(self, tmp_path, capsys): @@ -156,7 +205,7 @@ def test_a_source_with_no_data_yet_is_skipped_not_fatal(self, tmp_path, capsys): missing = Source( id="test/missing", display="Missing", src_path=tmp_path / "does-not-exist", - vault_subdir="data/test-missing", + vault_subdir="data/test/missing", ) present = self._make_source(tmp_path, "ok", file_count=20) @@ -527,7 +576,7 @@ def _source(self, tmp_path: Path) -> Source: src.mkdir() (src / "a.jpg").write_text("x") return Source(id="photos/library", display="Photos", - src_path=src, vault_subdir="data/photos-library") + src_path=src, vault_subdir="data/photos/library") def _fake_run(self, chflags_returncode: int): from types import SimpleNamespace diff --git a/tests/stacklets/test_backup_migrate.py b/tests/stacklets/test_backup_migrate.py new file mode 100644 index 00000000..0e3ae449 --- /dev/null +++ b/tests/stacklets/test_backup_migrate.py @@ -0,0 +1,181 @@ +"""Moving a vault to the current directory layout. + +A vault written by 0.3.0-beta.3 or earlier holds one flat directory per +source, `data/photos-library`. Current releases nest them under the +stacklet, `data/photos/library`. Syncs read both, so the rename is +something the household does once, when it suits them. + +The rename moves the directory, not its contents. That matters more than +it looks: every file on a vault carries the `uchg` flag that makes the +vault append-only, and a copy would both duplicate the data and leave +the original undeletable. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_ROOT / "stacklets" / "backup" / "cli")) + +from migrate import declared_sources, migrate_vault # noqa: E402 + +SOURCES = [("photos/library", "Photos"), ("messages/synapse", "Messages")] + + +def _flat(mount: Path, name: str, files: int = 1) -> Path: + d = mount / "data" / name + d.mkdir(parents=True) + for i in range(files): + (d / f"file-{i}.jpg").write_text("x") + return d + + +# ── The rename ───────────────────────────────────────────────────────────── + +class TestMigrateVault: + + def test_it_nests_a_flat_directory_under_its_stacklet(self, tmp_path): + _flat(tmp_path, "photos-library", files=3) + + moves = migrate_vault(tmp_path, SOURCES, dry_run=False) + + assert [(m.display, m.status) for m in moves] == [("Photos", "moved")] + assert not (tmp_path / "data" / "photos-library").exists() + assert len(list((tmp_path / "data" / "photos" / "library") + .glob("*.jpg"))) == 3 + + def test_a_vault_already_on_the_current_layout_is_left_alone(self, tmp_path): + (tmp_path / "data" / "photos" / "library").mkdir(parents=True) + + assert migrate_vault(tmp_path, SOURCES, dry_run=False) == [] + + def test_it_moves_every_source_it_finds(self, tmp_path): + _flat(tmp_path, "photos-library") + _flat(tmp_path, "messages-synapse") + + moves = migrate_vault(tmp_path, SOURCES, dry_run=False) + + assert {m.display for m in moves} == {"Photos", "Messages"} + assert (tmp_path / "data" / "photos" / "library").is_dir() + assert (tmp_path / "data" / "messages" / "synapse").is_dir() + + def test_a_dry_run_reports_without_touching_anything(self, tmp_path): + _flat(tmp_path, "photos-library") + + moves = migrate_vault(tmp_path, SOURCES, dry_run=True) + + assert [m.status for m in moves] == ["moved"] + assert (tmp_path / "data" / "photos-library").is_dir() + assert not (tmp_path / "data" / "photos").exists() + + def test_it_refuses_when_both_layouts_hold_data(self, tmp_path): + """No rename can reconcile this. The files are immutable, so the + two trees cannot be merged without unlocking them, and picking + one would quietly hide the other.""" + _flat(tmp_path, "photos-library") + nested = tmp_path / "data" / "photos" / "library" + nested.mkdir(parents=True) + (nested / "newer.jpg").write_text("x") + + moves = migrate_vault(tmp_path, SOURCES, dry_run=False) + + assert [m.status for m in moves] == ["occupied"] + assert (tmp_path / "data" / "photos-library" / "file-0.jpg").exists() + assert (nested / "newer.jpg").exists() + + def test_one_stuck_directory_does_not_stop_the_others(self, tmp_path, + monkeypatch): + _flat(tmp_path, "photos-library") + _flat(tmp_path, "messages-synapse") + + real_rename = Path.rename + + def rename(self, target): + if "photos" in str(self): + raise OSError("Read-only file system") + return real_rename(self, target) + + monkeypatch.setattr(Path, "rename", rename) + moves = migrate_vault(tmp_path, SOURCES, dry_run=False) + + assert {m.display: m.status for m in moves} == { + "Photos": "failed", "Messages": "moved", + } + + +# ── The immutable flag ───────────────────────────────────────────────────── + +@pytest.mark.skipif(sys.platform != "darwin", + reason="chflags is BSD-only; the rest of this file is portable") +class TestLockedFilesSurvive: + """The reason this is a rename and not a copy.""" + + def test_files_keep_their_immutable_flag_across_the_move(self, tmp_path): + flat = _flat(tmp_path, "photos-library", files=2) + subprocess.run(["find", str(flat), "-type", "f", + "-exec", "chflags", "uchg", "{}", "+"], check=True) + try: + moves = migrate_vault(tmp_path, SOURCES, dry_run=False) + + assert [m.status for m in moves] == ["moved"] + moved = tmp_path / "data" / "photos" / "library" / "file-0.jpg" + assert "uchg" in _flags(moved) + with pytest.raises(PermissionError): + moved.write_text("tampered") + finally: + subprocess.run(["chflags", "-R", "nouchg", str(tmp_path)], + check=False) + + +def _flags(path: Path) -> str: + out = subprocess.run(["ls", "-lO", str(path)], + capture_output=True, text=True, check=True) + return out.stdout + + +# ── What gets migrated ───────────────────────────────────────────────────── + +class TestDeclaredSources: + """Archives and snapshots both own a directory under `data/`, so a + migration has to consider both.""" + + def test_it_covers_archives_and_snapshots_of_enabled_stacklets(self, + tmp_path): + repo = tmp_path / "repo" + (repo / "stacklets" / "messages").mkdir(parents=True) + (repo / "stacklets" / "messages" / "stacklet.toml").write_text( + 'name = "Messages"\n' + '[[backup.archive]]\n' + 'name = "media"\n' + 'path = "{data_dir}/messages/media"\n' + '[[backup.snapshot]]\n' + 'name = "synapse"\n' + 'postgres = { container = "c", database = "d", user = "u" }\n' + ) + instance = tmp_path / "instance" + (instance / ".stack").mkdir(parents=True) + (instance / ".stack" / "messages.setup-done").touch() + + found = declared_sources(repo, instance, tmp_path / "data") + + assert found == [("messages/media", "Messages"), + ("messages/synapse", "Messages")] + + def test_a_disabled_stacklet_contributes_nothing(self, tmp_path): + """Its directory stays flat and stays readable, which is the + point of keeping both layouts working.""" + repo = tmp_path / "repo" + (repo / "stacklets" / "photos").mkdir(parents=True) + (repo / "stacklets" / "photos" / "stacklet.toml").write_text( + 'name = "Photos"\n[[backup.archive]]\nname = "library"\n' + 'path = "{data_dir}/photos"\n' + ) + instance = tmp_path / "instance" + (instance / ".stack").mkdir(parents=True) + + assert declared_sources(repo, instance, tmp_path / "data") == [] diff --git a/tests/stacklets/test_backup_orchestrator.py b/tests/stacklets/test_backup_orchestrator.py index 59a91834..d5c428f8 100644 --- a/tests/stacklets/test_backup_orchestrator.py +++ b/tests/stacklets/test_backup_orchestrator.py @@ -80,7 +80,7 @@ def test_finds_archive_entries_from_enabled_stacklets(self, tmp_path): assert s.id == "photos/library" assert s.display == "Photos" assert s.src_path == Path("/var/famstack-data/photos/library/library") - assert s.vault_subdir == "data/photos-library" + assert s.vault_subdir == "data/photos/library" # Replaced by a baseline the engine derives itself. assert s.rolling is False @@ -121,7 +121,7 @@ def test_multiple_archives_per_stacklet(self, tmp_path): ) sources = discover_archive_sources(tmp_path, tmp_path, Path("/d")) assert [s.id for s in sources] == ["photos/library", "photos/shared"] - assert [s.vault_subdir for s in sources] == ["data/photos-library", "data/photos-shared"] + assert [s.vault_subdir for s in sources] == ["data/photos/library", "data/photos/shared"] def test_template_variable_renders(self, tmp_path): # {data_dir} must expand to whatever the orchestrator was given. @@ -229,12 +229,12 @@ def test_single_record_formatted(self): sources = [SourceRecord( id="photos/library", display="Photos", src_path=Path("/var/famstack-data/photos/library/library"), - vault_subdir="data/photos-library", + vault_subdir="data/photos/library", )] env = serialize_sources_env(sources) assert env == ( "photos/library|Photos|/var/famstack-data/photos/library/library|" - "data/photos-library|0" + "data/photos/library|0" ) def test_a_rolling_source_is_flagged_for_the_engine(self): @@ -242,7 +242,7 @@ def test_a_rolling_source_is_flagged_for_the_engine(self): has to know so it does not read that as data loss.""" sources = [SourceRecord( id="messages/synapse", display="Messages", src_path=Path("/a"), - vault_subdir="data/messages-synapse", rolling=True, + vault_subdir="data/messages/synapse", rolling=True, )] assert serialize_sources_env(sources).endswith("|1") diff --git a/tests/stacklets/test_backup_snapshot.py b/tests/stacklets/test_backup_snapshot.py index b40056c2..b68fb405 100644 --- a/tests/stacklets/test_backup_snapshot.py +++ b/tests/stacklets/test_backup_snapshot.py @@ -66,7 +66,7 @@ def test_it_writes_one_dated_tarball(self, tmp_path): path = take_snapshot(_spec(tmp_path), out, dump=_fake_dump()) # Namespaced per stacklet+name so two stacklets cannot collide. - assert path.parent == out / "messages-synapse" + assert path.parent == out / "messages" / "synapse" assert path.suffixes[-2:] == [".tar", ".gz"] assert path.name.startswith("synapse-") assert path.exists() @@ -159,8 +159,8 @@ class TestSnapshotSource: def test_the_output_directory_becomes_a_source(self, tmp_path): src = snapshot_source(_spec(tmp_path), tmp_path / "snapshots") assert src.id == "messages/synapse" - assert src.src_path == tmp_path / "snapshots" / "messages-synapse" - assert src.vault_subdir == "data/messages-synapse" + assert src.src_path == tmp_path / "snapshots" / "messages" / "synapse" + assert src.vault_subdir == "data/messages/synapse" def test_it_is_marked_rolling(self, tmp_path): """The directory is pruned to a fixed size, so the engine's