From 6997f67475857c89ed9c6e7162398daeb34fc4b0 Mon Sep 17 00:00:00 2001 From: Alexander Nguyen Date: Fri, 22 May 2026 14:42:35 -0700 Subject: [PATCH 01/73] feat: operator-free per-file quiescence-driven NAS sync Replace the equipment-written completeness signal (sentinel file / manifest) with a background quiescence poller that syncs each file independently once it has been observed unchanged for a settle window (default 10 min) -- no equipment cooperation, no operator marking. - config: drop per-equipment completeness_signal/sentinel/manifest; add sync.quiescence_minutes / ignore_globs / poll_interval_seconds. The Add-Equipment wizard goes 5 -> 4 steps. - new per-run sync_state.json (per-file synced_signature / verified_at / keep_local, plus cleared_at); ingest.json / IngestWriter / IngestState removed. - QuiescenceSyncPoller replaces StagingWatcher; discovers both staging-area and nas-mode runs. - per-file sync through the SyncQueue (new files column), transports gain --files-from, verifier gains a subset filter, per-file verify reconciliation credits each verified file into sync_state.json. - cleanup honors per-file keep-local and is symlink-safe; the run rollup (syncing/synced/cleared) is derived from sync_state.json. - GUI: per-file sync status including "On NAS" tombstones for cleared runs, plus a keep-local toggle and its API endpoint. - wire the sync pipeline that was never actually started in production -- the poller and the NASSyncClient worker now run via the app lifespan. Also includes earlier same-session fixes: the Add-Equipment wizard's reactive Next button, persistent multi-step state, and working confirm; the encrypted-at-rest keyring fallback (EXLAB_WIZARD_SECRET_PASSPHRASE); and a repo-wide lims_password_present accessor. Design: docs/superpowers/specs/2026-05-21-operator-free-per-file-nas-sync-design.md Co-Authored-By: Claude Opus 4.7 --- .gitignore | 5 + docs/UX_INTERACTIONS.md | 4 - docs/source/user_guide/01_settings.md | 11 +- ...-operator-free-per-file-nas-sync-design.md | 259 ++++++ src/exlab_wizard/api/_dependencies.py | 21 + src/exlab_wizard/api/app.py | 27 +- src/exlab_wizard/api/health.py | 4 +- src/exlab_wizard/api/routers/browse.py | 327 ++++++-- src/exlab_wizard/api/routers/config.py | 31 +- src/exlab_wizard/api/routers/staging.py | 199 +++-- src/exlab_wizard/api/schemas.py | 61 +- src/exlab_wizard/api/setup.py | 4 +- src/exlab_wizard/cache/ingest_writer.py | 206 ----- src/exlab_wizard/cache/sync_state_schema.py | 62 ++ src/exlab_wizard/cache/sync_state_writer.py | 272 ++++++ src/exlab_wizard/config/models.py | 62 +- src/exlab_wizard/constants/__init__.py | 14 +- src/exlab_wizard/constants/enums.py | 32 +- src/exlab_wizard/constants/filenames.py | 5 +- src/exlab_wizard/constants/schema_versions.py | 6 +- src/exlab_wizard/controller/creation.py | 12 +- src/exlab_wizard/orchestrator/__init__.py | 27 +- src/exlab_wizard/orchestrator/_scan.py | 68 +- src/exlab_wizard/orchestrator/cleanup.py | 209 ----- .../orchestrator/quiescence_poller.py | 358 ++++++++ .../orchestrator/staging_clear.py | 80 ++ .../orchestrator/staging_query.py | 158 ++-- .../orchestrator/staging_watcher.py | 546 ------------- src/exlab_wizard/paths.py | 7 - src/exlab_wizard/sync/nas_client.py | 285 ++++++- src/exlab_wizard/sync/queue.py | 80 +- src/exlab_wizard/sync/run_delete.py | 116 +++ src/exlab_wizard/sync/transports/rclone.py | 8 + src/exlab_wizard/sync/transports/rsync_ssh.py | 12 + src/exlab_wizard/sync/verifier.py | 42 +- src/exlab_wizard/tray/dependencies.py | 143 +++- src/exlab_wizard/tray/main.py | 2 - src/exlab_wizard/ui/components/file_list.py | 59 +- .../ui/components/metadata_pane.py | 1 - .../ui/components/sync_status_icon.py | 32 +- src/exlab_wizard/ui/components/tree.py | 25 +- src/exlab_wizard/ui/equipment_form.py | 14 - src/exlab_wizard/ui/mount.py | 291 +++++-- src/exlab_wizard/ui/pages/settings.py | 39 +- src/exlab_wizard/ui/pages/staging.py | 28 +- src/exlab_wizard/ui/pages/wizard_equipment.py | 237 +++--- tests/e2e/_test_app.py | 122 ++- .../e2e/page_objects/wizard_equipment_page.py | 17 +- tests/e2e/test_flow_00_fresh_install_setup.py | 10 +- tests/e2e/test_flow_00_full_lifecycle.py | 10 +- .../test_flow_05_browse_view_sync_icons.py | 10 +- tests/e2e/test_flow_09_orchestrator.py | 19 +- tests/e2e/test_flow_16_add_equipment.py | 44 +- tests/e2e/test_flow_24_context_menus.py | 50 +- .../test_flow_26_equipment_wizard_persist.py | 98 +++ tests/e2e/ux_catalog.py | 32 - tests/fixtures/configs/complete.yaml | 4 - .../fixtures/configs/incomplete_no_lims.yaml | 2 - .../fixtures/configs/incomplete_no_paths.yaml | 2 - tests/integration/api/test_full_flow.py | 2 - .../controller/test_creation_flow.py | 2 - tests/integration/test_nas_sync.py | 165 +++- .../test_orchestrator_lifecycle.py | 400 +++------ .../integration/test_schema_major_mismatch.py | 20 +- tests/unit/api/test_browse.py | 345 +++++--- tests/unit/api/test_config_router.py | 6 - tests/unit/api/test_health.py | 2 - tests/unit/api/test_operations.py | 2 - tests/unit/api/test_problems.py | 2 - tests/unit/api/test_schemas.py | 106 +-- tests/unit/api/test_sessions.py | 2 - tests/unit/api/test_setup.py | 2 - tests/unit/api/test_staging_router.py | 400 ++++++--- tests/unit/cache/test_ingest_writer.py | 509 ------------ tests/unit/cache/test_sync_state_writer.py | 328 ++++++++ tests/unit/config/test_loader.py | 8 - tests/unit/config/test_models.py | 129 ++- tests/unit/constants/test_enums.py | 37 +- tests/unit/constants/test_filenames.py | 8 +- tests/unit/constants/test_schema_versions.py | 10 +- tests/unit/orchestrator/test_cleanup.py | 395 --------- .../orchestrator/test_quiescence_poller.py | 369 +++++++++ tests/unit/orchestrator/test_scan.py | 2 +- tests/unit/orchestrator/test_staging_query.py | 271 +++--- .../unit/orchestrator/test_staging_watcher.py | 773 ------------------ tests/unit/sync/test_nas_client.py | 110 ++- tests/unit/sync/test_nas_client_extra.py | 449 +++++++++- tests/unit/sync/test_queue.py | 31 + tests/unit/sync/test_transports.py | 74 ++ tests/unit/sync/test_verifier.py | 38 + tests/unit/test_paths.py | 2 - tests/unit/tray/test_dependencies.py | 54 +- tests/unit/ui/test_components.py | 39 +- tests/unit/ui/test_dynamic_form.py | 26 +- tests/unit/ui/test_file_list.py | 81 ++ tests/unit/ui/test_mount.py | 109 +-- tests/unit/ui/test_staging_page.py | 30 +- tests/unit/ui/test_wizard_equipment.py | 22 +- 98 files changed, 5740 insertions(+), 4531 deletions(-) create mode 100644 docs/superpowers/specs/2026-05-21-operator-free-per-file-nas-sync-design.md delete mode 100644 src/exlab_wizard/cache/ingest_writer.py create mode 100644 src/exlab_wizard/cache/sync_state_schema.py create mode 100644 src/exlab_wizard/cache/sync_state_writer.py delete mode 100644 src/exlab_wizard/orchestrator/cleanup.py create mode 100644 src/exlab_wizard/orchestrator/quiescence_poller.py create mode 100644 src/exlab_wizard/orchestrator/staging_clear.py delete mode 100644 src/exlab_wizard/orchestrator/staging_watcher.py create mode 100644 src/exlab_wizard/sync/run_delete.py create mode 100644 tests/e2e/test_flow_26_equipment_wizard_persist.py delete mode 100644 tests/unit/cache/test_ingest_writer.py create mode 100644 tests/unit/cache/test_sync_state_writer.py delete mode 100644 tests/unit/orchestrator/test_cleanup.py create mode 100644 tests/unit/orchestrator/test_quiescence_poller.py delete mode 100644 tests/unit/orchestrator/test_staging_watcher.py diff --git a/.gitignore b/.gitignore index 3d14c6a..0c3ea67 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,11 @@ state/ *.tmp .tmp/ +# Local secrets / env (never commit) +.env +.env.* +*.secret + # pre-commit cache .pre-commit-cache/ diff --git a/docs/UX_INTERACTIONS.md b/docs/UX_INTERACTIONS.md index 762a618..6d44b03 100644 --- a/docs/UX_INTERACTIONS.md +++ b/docs/UX_INTERACTIONS.md @@ -43,9 +43,6 @@ flow test. | `/settings` | `settings-equipment-label` | input | Type the equipment label | Provides the EquipmentConfig.label for the new entry. | | `/settings` | `settings-equipment-local-root` | input | Type the equipment local root | Provides the EquipmentConfig.local_root for the new entry. | | `/settings` | `settings-equipment-nas-root` | input | Type the equipment NAS root | Provides the EquipmentConfig.nas_root for the new entry. | -| `/settings` | `settings-equipment-signal` | radio | Pick the completeness signal (sentinel_file / manifest) | Swaps the filename field between sentinel and manifest. | -| `/settings` | `settings-equipment-sentinel` | input | Type the sentinel filename | Sets the sentinel_file completeness signal filename. | -| `/settings` | `settings-equipment-manifest` | input | Type the manifest filename | Sets the manifest completeness signal filename. | | `/settings` | `settings-equipment-transport` | radio | Pick the transport (rclone / rsync_ssh) | Swaps the transport fieldset between rclone and rsync_ssh. | | `/settings` | `settings-equipment-rclone-remote` | input | Type the rclone remote | Sets the rclone transport remote for the new entry. | | `/settings` | `settings-equipment-rclone-path` | input | Type the rclone remote path | Sets the rclone transport remote path for the new entry. | @@ -109,7 +106,6 @@ flow test. | `/wizard/equipment` | `wizard-equipment-label` | input | Type the equipment label | Sets the human-readable equipment label. | | `/wizard/equipment` | `wizard-equipment-local-root` | input | Type the equipment's local root path | Sets where this device acquires runs on disk. | | `/wizard/equipment` | `wizard-equipment-sync-mode` | radio | Pick 'nas' or 'stage' sync mode | Swaps the transport sub-form between NAS-direct and stage-push. | -| `/wizard/equipment` | `wizard-equipment-signal` | radio | Pick 'sentinel_file' or 'manifest' completeness signal | Swaps the filename input between sentinel and manifest naming. | | `/wizard/equipment` | `wizard-equipment-confirm` | button | Click 'Confirm' on the review step | Posts the assembled EquipmentConfig via POST /config/equipment. | | `/wizard/equipment` | `wizard-equipment-cancel` | button | Click 'Cancel' on any wizard step | Discards the wizard and returns to /main. | diff --git a/docs/source/user_guide/01_settings.md b/docs/source/user_guide/01_settings.md index 95dba73..f68fc0f 100644 --- a/docs/source/user_guide/01_settings.md +++ b/docs/source/user_guide/01_settings.md @@ -32,9 +32,9 @@ order. See section 02 §3.6 for the authoritative contract and section The Settings dialog edits equipment that already exists; registering a **new** device uses the dedicated Add-Equipment wizard, opened from the main-window toolbar's *Add Equipment* button -(`data-testid="toolbar-add-equipment"`). The wizard walks five steps -- -identity, paths, sync mode, completeness signal, and a review step -- -then posts the assembled `EquipmentConfig` to the configuration router. +(`data-testid="toolbar-add-equipment"`). The wizard walks four steps -- +identity, paths, sync mode, and a review step -- then posts the +assembled `EquipmentConfig` to the configuration router. Editing or removing a registered device stays in the Settings dialog's Equipment List section, which the file-explorer tree context menus deep-link into (Redesign decision 4A). @@ -42,9 +42,8 @@ deep-link into (Redesign decision 4A). 1. **Open the wizard.** Click *Add Equipment* on the main-window toolbar to navigate to `/wizard/equipment`. 2. **Step through the wizard.** Supply the equipment identity - (`data-testid="wizard-equipment-id"`), the local and NAS paths, the - sync mode (`nas` or `stage`), and the completeness signal - (`sentinel_file` or `manifest`). + (`data-testid="wizard-equipment-id"`), the local and NAS paths, and + the sync mode (`nas` or `stage`). 3. **Confirm.** The Confirm button on the review step (`data-testid="wizard-equipment-confirm"`) registers the device and returns to the main window. diff --git a/docs/superpowers/specs/2026-05-21-operator-free-per-file-nas-sync-design.md b/docs/superpowers/specs/2026-05-21-operator-free-per-file-nas-sync-design.md new file mode 100644 index 0000000..945523b --- /dev/null +++ b/docs/superpowers/specs/2026-05-21-operator-free-per-file-nas-sync-design.md @@ -0,0 +1,259 @@ +# Operator-free, per-file quiescence-driven NAS sync — design + +- **Date:** 2026-05-21 +- **Status:** Approved (design); implementation plan pending +- **Affects:** orchestrator staging, NAS sync, config schema, Add-Equipment wizard, main GUI + +## Context + +Today a staged run is promoted from `staging` to `complete` when the +orchestrator observes a *completeness signal* — a sentinel file the +**equipment machine** writes, or a manifest the equipment emits +(Design Spec §13.5). Completeness, the sentinel/manifest filenames, and +the choice between them are configured **per equipment** in +`config.yaml` and collected by step 4 of the Add-Equipment wizard. + +This couples the orchestrator to equipment-side cooperation and to a +fixed per-equipment policy. It also offers no protection against +syncing a file that is still being written: the post-transfer SHA-256 +verifier (`sync/verifier.py`) catches *wire* corruption but not a +partial *source* file — a half-written file copies and hashes +consistently on both ends and passes verification. + +## Goals + +- Decide a file is ready to sync with **zero equipment cooperation**. +- Never transfer a file that is still being written. +- Remove operator/equipment involvement from the sync trigger entirely: + no marking, no sentinel, no manifest. +- Sync each file **independently**, as soon as it is safe. + +## Non-goals + +- Changing the equipment → staging transport (outside the app, §13.6). +- Operator "mark complete / incomplete" actions (considered and + dropped — quiescence is the trigger, so marking is unnecessary). +- Automatic completeness *detection* by sentinel/manifest (removed). + +## The model + +A background **quiescence poller** sweeps every run directory pending +NAS sync — **both** orchestrator staging-area runs **and** runs +acquired directly on `nas`-mode equipment — on an interval. One +unified poller is the single sync trigger for every run that must reach +the NAS. A file becomes eligible to sync when **all** hold: + +1. the poller has **observed** its `(size, mtime)` unchanged across its + own consecutive sweeps spanning ≥ the settle threshold + (`sync.quiescence_minutes`, default **10**). Eligibility is measured + from the poller's own observations, *not* the absolute age of + `mtime` — transports (`rsync -t`, `rclone`) preserve the source + `mtime`, so a file can land in staging already showing an old + timestamp. The poller must therefore persist or carry forward a + prior-sweep snapshot to compare against. +2. it does not match any `sync.ignore_globs` entry (default + `["*.partial", "*.tmp"]`); +3. it is not already synced *at its current state* — a file modified + after a prior successful sync becomes eligible again once it + re-settles. + +Eligible files are synced to the NAS independently (file-level +granularity). There is no run-level "complete" gate. + +## Config changes (`config/models.py`) + +**Remove from `EquipmentConfig`:** + +- `completeness_signal`, `sentinel_filename`, `manifest_filename`; +- the `_completeness_signal_requires_matching_filename` model validator; +- the `_serialize_completeness_signal` field serializer. + +**Remove the `CompletenessSignal` enum** (`constants/enums.py`) — no +remaining consumer once the watcher's signal logic is gone. + +**Add to `SyncConfig`:** + +- `quiescence_minutes: int = 10` (ge ≥ 1) — the per-file settle window. +- `ignore_globs: list[str] = ["*.partial", "*.tmp"]` — names skipped + from eligibility (in-progress transport temp files). +- `poll_interval_seconds: int = 120` — how often the poller sweeps. + Coarse by design: with a 10-minute settle window there is no value + in sweeping faster. + +These are **global** settings, not per-equipment — completeness policy +is no longer an equipment attribute. + +## Run lifecycle — rollup (Approach 1) + +`IngestState` collapses from five states (`staging`, `complete`, +`sync_queued`, `sync_verified`, `cleared`) to the milestones that are +genuinely monotonic plus a derived rollup: + +- `SYNCING` — at least one non-ignored file is unsynced/unverified. +- `SYNCED` — every non-ignored file is verified on the NAS. +- `CLEARED` — the run's staging copy has been cleaned up. + +A run's `SYNCING`/`SYNCED` status is **derived on read** from +`sync_state.json` (below); it is never an appended history entry, +because it can oscillate (a `SYNCED` run whose file is modified again +returns to `SYNCING`). + +## Per-file sync state — `sync_state.json` + +A new per-run file `/.exlab-wizard/sync_state.json`, written by the +orchestrator only. It is a **freely-mutable current-state map** (not +append-only): relative file path → record: + +``` +{ + "": { + "synced_signature": [size, mtime], // (size, mtime) at last successful sync; null if never synced + "verified_at": "", // null until SHA-256 verified + "keep_local": false + }, + ... +} +``` + +- `synced_signature` answers both "already synced?" and "modified since + sync?" — if the file's current `(size, mtime)` differs, it is + re-eligible once it re-settles. +- This file is the source of truth the GUI's existing + `sync_status_icon` reads for per-file status. + +## `ingest.json` contract (risk 2 resolution) + +`ingest.json` keeps its append-only `history`, but its entries are +**milestone events**, not rollup states: `created` at run bootstrap and +`cleared` at cleanup. It no longer stores a `syncing`/`synced` +`current_state`. Consequently `CLEARED` is the only `IngestState` +member ever persisted; `SYNCING` and `SYNCED` exist solely as the +computed rollup and are never written to `history`. All churny +per-file and rollup data lives in the freely-mutable `sync_state.json`, +and the run-level rollup is computed on read from it whenever the GUI +or cleanup needs it. This preserves the append-only history contract +without spamming it with rollup oscillation. + +## `keep-local` (per-file) + +A per-file boolean in `sync_state.json`. A `keep_local` file **still +syncs** to the NAS; it is **excluded from cleanup deletion**. The +operator toggles it from the file-list context menu, but +`sync_state.json` has a **single writer — the orchestrator** — so the +context-menu action calls a backend API endpoint that applies the flag; +the GUI never writes `sync_state.json` directly. + +## GUI per-file display state + +The GUI file list for a run is sourced from `sync_state.json` (the +durable per-file record), unioned with any local files not yet +recorded. Because `sync_state.json` lives in `/.exlab-wizard/` and +**survives cleanup**, the operator never loses visibility — a fully +cleared run still expands to show every file as an "On NAS" tombstone. + +Each file resolves to one display state: + +| In `sync_state.json` | Local file on disk | GUI state | +|---|---|---| +| not recorded yet | present | **Acquiring** — new, still settling | +| recorded, `verified_at` null | present | **Syncing** — settled / in transfer | +| recorded, `verified_at` set | present | **Synced** — on NAS, local copy still here | +| recorded, `verified_at` set | absent | **On NAS** — tombstone, local copy cleared | +| `keep_local` true | present | **Kept local** badge, plus its sync state | + +The run-node rollup (`SYNCING` / `SYNCED` / `CLEARED`) is derived from +these per-file states. + +## Failure handling + +A file whose transfer or SHA-256 verification fails keeps +`synced_signature` and `verified_at` null in `sync_state.json`, so it +stays eligible and is retried on the next sweep, bounded by +`sync.retry_attempts`. After retries are exhausted the file is surfaced +as a per-file error in the GUI (existing problems/sync-status +machinery) and the run rollup stays `SYNCING`. + +## Cleanup — rollup + +Per-run, trigger style unchanged: once a run is `SYNCED` and +`retain_hours` has elapsed, clear it — delete every file **except** +those flagged `keep_local` and the `.exlab-wizard/` metadata directory. +`StagingCleanupMode.MANUAL` still never auto-clears. + +## Transport batching (risk 1 resolution) + +Per-file eligibility must not become per-file transport invocations. +The poller groups a sweep's eligible files **by run** and emits **one +transport job per run**, carrying that run's eligible-file list. Runs +cannot be batched together — each has its own `local_root → nas_root` +src/dst pair — so per-run is the natural and maximal batch. + +- `sync/queue.py` job payload becomes `(run_path, [relative paths])`. +- `sync/transports/rclone.py` and `rsync_ssh.py` gain a files-from + mode: write the list to a temp file, pass `--files-from`. Both tools + support this natively (`rclone copy --files-from`, `rsync + --files-from`). + +After each per-run batched transfer the existing `sync/verifier.py` +SHA-256 pass runs over the transferred files; each file's `verified_at` +in `sync_state.json` is set when its hash is confirmed against the NAS +copy. A file counts toward the `SYNCED` rollup only once `verified_at` +is set. + +## Components touched + +- **New `QuiescenceSyncPoller`** — the single sync trigger for **every** + run pending NAS sync, orchestrator-staged *and* `nas`-mode. It + supersedes the run-state machine in + `orchestrator/staging_watcher.py`'s `evaluate_run`: sweep → per-file + eligibility → group by run → enqueue → on verify, update + `sync_state.json`. The existing `StagingWatcher`'s sentinel/manifest + watching is removed; its run-discovery for the staging area is reused + and extended to also discover `nas`-mode run directories. +- **`sync/queue.py` + transports** — per-run file-list jobs (above). +- **`config/models.py`, `constants/enums.py`** — config + enum changes. +- **`api/schemas.py`** — `IngestJson` state set; new `sync_state.json` + schema/struct. +- **API** — a `keep_local` toggle endpoint (single-writer; see + *`keep-local`* above). +- **Add-Equipment wizard** (`ui/pages/wizard_equipment.py`) — drop + step 4 "Completeness signal"; `EQUIPMENT_WIZARD_STEPS` 5 → 4; update + `EquipmentWizardState`, `can_advance`, `_STEP_RENDERERS`, + `assemble_equipment_config`/`build_equipment_config`. +- **GUI** — run tree shows the `syncing`/`synced` rollup; file list + is sourced from `sync_state.json` (per *GUI per-file display state*), + keeps per-file sync icons including "On NAS" tombstones, and gains a + "Keep local" context-menu toggle. + +## Testing + +- **Unit:** quiescence eligibility (settle window boundary, ignore-glob, + modified-since-sync re-eligibility); rollup derivation from + `sync_state.json`; `sync_state.json` read/write; cleanup skipping + `keep_local` files; `EquipmentConfig` no longer accepts the removed + fields. +- **Integration:** poller sweep → per-run batched enqueue → verify → + `sync_state.json` updated → rollup flips to `SYNCED`; a re-modified + file flips the rollup back to `SYNCING`; the poller discovers and + drives both a staging-area run and a `nas`-mode run; a transfer + failure leaves the file eligible and retries. +- **E2E:** Add-Equipment wizard is 4 steps and persists (extends the + existing flow_16 / flow_26 coverage); file-list "Keep local" toggle; + a cleared run still lists its files as "On NAS" tombstones. + +## Migration + +Existing `config.yaml` files carry per-equipment `completeness_signal` +etc. The config loader must tolerate and drop these removed keys on +load (they currently use `extra="forbid"`, so a one-time prune or a +pre-validation migration step is required) so an upgraded install does +not fail to boot. + +## Open questions / out of scope + +- Whether a stalled writer that pauses > `quiescence_minutes` mid-file + can produce a false-stable read. Accepted risk for this iteration; + the manifest-based check that would close it was explicitly declined + to avoid re-introducing equipment coupling. +- Cross-run global batching is intentionally not pursued (impossible + given per-run src/dst roots). diff --git a/src/exlab_wizard/api/_dependencies.py b/src/exlab_wizard/api/_dependencies.py index 0c46339..e03aca4 100644 --- a/src/exlab_wizard/api/_dependencies.py +++ b/src/exlab_wizard/api/_dependencies.py @@ -21,11 +21,32 @@ from fastapi import HTTPException, Request, status __all__ = [ + "lims_password_present", "require_controller", "require_deps", ] +def lims_password_present(deps: Any) -> bool: + """Return whether a LIMS password is stored -- the repo-wide reader. + + Every surface that asks "is the LIMS keyring password set?" -- the + settings credential field, the setup-state evaluator, the + section-completion gate -- routes through here so the default and + the ``deps is None`` handling stay identical instead of each call + site open-coding its own ``getattr(deps, "keyring_password_present", + ...)`` with its own default. + + ``deps`` is typed ``Any`` because callers hold it loosely + (``AppDependencies`` in production, mocks in tests, ``None`` before + wiring). A ``None`` or attribute-less ``deps`` means nothing is + wired yet, so the password cannot be present -- hence ``False``. + """ + if deps is None: + return False + return bool(getattr(deps, "keyring_password_present", False)) + + def require_deps(request: Request) -> Any: """Return ``app.state.dependencies`` or raise a structured 503. diff --git a/src/exlab_wizard/api/app.py b/src/exlab_wizard/api/app.py index 17eff8a..63966f7 100644 --- a/src/exlab_wizard/api/app.py +++ b/src/exlab_wizard/api/app.py @@ -197,8 +197,13 @@ class AppDependencies: lims_client: Any = None nas_sync: Any = None session_store: Any = None - ingest_writer: Any = None - staging_watcher: Any = None + quiescence_poller: Any = None + # Orchestrator-only ``sync_state.json`` writer + # (:class:`exlab_wizard.cache.sync_state_writer.SyncStateWriter`). The + # quiescence poller reads it; the NAS-sync client writes per-file + # verify reconciliation into it (operator-free per-file NAS sync, + # 2026-05-21). + sync_state_writer: Any = None # OS-keyring store (:class:`exlab_wizard.lims.keyring_store.KeyringStore`). # The settings dialog's credential fields write the LIMS password # straight to this at click time (Frontend Spec §7.3, §7.4.1). @@ -257,9 +262,27 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: _audit_loop(deps, audit_interval_seconds), name="exlab-audit-loop", ) + # Operator-free per-file NAS sync (2026-05-21): the NASSyncClient + # owns the durable queue + transport worker; ``init()`` opens the + # queue DB and spawns the worker task -- ``enqueue`` itself fails + # without it. The quiescence poller is the auto-sync trigger and + # must start *after* the client it feeds. Both are ``None`` when + # no staging_root / nas-mode equipment is configured. + if deps.nas_sync is not None: + with contextlib.suppress(Exception): + await deps.nas_sync.init() + if deps.quiescence_poller is not None: + with contextlib.suppress(Exception): + await deps.quiescence_poller.start() try: yield finally: + if deps.quiescence_poller is not None: + with contextlib.suppress(asyncio.CancelledError, Exception): + await deps.quiescence_poller.stop() + if deps.nas_sync is not None: + with contextlib.suppress(asyncio.CancelledError, Exception): + await deps.nas_sync.close() if deps.audit_task is not None and not deps.audit_task.done(): deps.audit_task.cancel() with contextlib.suppress(asyncio.CancelledError, Exception): diff --git a/src/exlab_wizard/api/health.py b/src/exlab_wizard/api/health.py index 6a7097d..0e8864e 100644 --- a/src/exlab_wizard/api/health.py +++ b/src/exlab_wizard/api/health.py @@ -22,8 +22,8 @@ from exlab_wizard.api.setup import compute_setup_state from exlab_wizard.constants import ( CREATION_JSON_VERSION, - INGEST_JSON_VERSION, README_FIELDS_JSON_VERSION, + SYNC_STATE_JSON_VERSION, ) from exlab_wizard.logging import get_logger @@ -60,7 +60,7 @@ async def get_health(request: Request) -> HealthResponse: schema_versions={ "creation_json": CREATION_JSON_VERSION, "readme_fields_json": README_FIELDS_JSON_VERSION, - "ingest_json": INGEST_JSON_VERSION, + "sync_state_json": SYNC_STATE_JSON_VERSION, }, components=components, setup_state=setup_state_value, diff --git a/src/exlab_wizard/api/routers/browse.py b/src/exlab_wizard/api/routers/browse.py index 9ffc82c..ca35f9e 100644 --- a/src/exlab_wizard/api/routers/browse.py +++ b/src/exlab_wizard/api/routers/browse.py @@ -130,7 +130,16 @@ class TreeResponse(BaseModel): class FolderEntry(BaseModel): - """One row in the new ``GET /folder/{path}`` response. Redesign §4.3 / §5.""" + """One row in the new ``GET /folder/{path}`` response. Redesign §4.3 / §5. + + Operator-free per-file NAS sync design (2026-05-21): ``sync_status`` + is one of the five GUI display states -- ``acquiring`` / ``syncing`` / + ``synced`` / ``on_nas`` -- sourced from the run's ``sync_state.json``. + A ``tombstone`` entry (``is_dir=False``, ``size_bytes=None``, + ``modified_iso=None``) represents a file present in ``sync_state.json`` + but absent on disk -- a cleared run still lists its files as "On NAS". + ``keep_local`` carries the file's keep-local flag for the badge. + """ model_config = ConfigDict(extra="forbid") @@ -140,6 +149,8 @@ class FolderEntry(BaseModel): size_bytes: int | None = None modified_iso: str | None = None sync_status: str | None = None + keep_local: bool = False + tombstone: bool = False class FolderResponse(BaseModel): @@ -169,12 +180,12 @@ class RunDetail(BaseModel): class RunLogEntry(BaseModel): """One row in the ``GET /run/{path}/log`` response. - The orchestrator does not write per-run log files; the "log" for a - run is the state-transition history of its ``ingest.json``. Each - history entry carries at minimum ``state`` and ``at``; transient - extras (``host``, ``files_received`` on ``complete``, etc.) are - forwarded as a free-form payload so the UI can render whatever the - orchestrator recorded. + The orchestrator does not write per-run log files. After the + operator-free per-file NAS sync redesign (2026-05-21) removed + ``ingest.json``, the per-run "log" is derived from the run's + sync-queue job: each entry carries the queue ``state`` and ``at`` + timestamp, with queue extras (``attempts``, ``last_error`` ...) + forwarded as a free-form ``payload``. """ model_config = ConfigDict(extra="forbid") @@ -243,69 +254,34 @@ async def get_folder(request: Request, folder_path: str) -> FolderResponse: response_model=RunLogResponse, dependencies=[Depends(setup_state_gate)], ) - async def get_run_log(run_path: str) -> RunLogResponse: - """Return the staged run's ``ingest.json`` history as a log. + async def get_run_log(request: Request, run_path: str) -> RunLogResponse: + """Return the staged run's sync-queue job state as a log. - Redesign §4.6 View-log surface. The orchestrator does not write - per-run log files; the lifecycle history in - ``/.exlab-wizard/ingest.json`` IS the per-run log. Returns - 404 when ingest.json doesn't exist (the run hasn't been staged - yet or has been cleared) and 422 on a parse failure. The - ``current_state`` field mirrors the most recent history entry - so the UI can show a header before iterating. + Redesign §4.6 View-log surface. The operator-free per-file NAS + sync redesign (2026-05-21) removed ``ingest.json``; the per-run + log is now derived from the run's sync-queue job. Returns 404 + when the run directory does not exist (never staged, or cleared + without a surviving record). A run with no queue job yet returns + an empty ``history`` and ``current_state == "none"``. Declared above the ``GET /run/{run_path:path}`` matcher because FastAPI matches routes in declaration order and the ``:path`` converter would otherwise swallow the trailing ``/log``. """ path = Path(run_path) - from exlab_wizard.api.schemas import IngestJson as _IngestJson - from exlab_wizard.constants import INGEST_JSON_NAME as _INGEST_JSON_NAME - - ingest_path = path / CACHE_DIR_NAME / _INGEST_JSON_NAME - if not ingest_path.exists(): - # Reuse ``session_not_found`` (same allowlist as the run- - # detail endpoint at GET /run/{path}) rather than minting a - # new code; semantically the run record is missing in both - # cases (creation.json there, ingest.json here). + if not path.exists(): # noqa: ASYNC240 -- one-shot existence stat raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={ "code": "session_not_found", - "message": f"ingest.json not found at {ingest_path}", - }, - ) - try: - payload = read_msgspec_json(ingest_path, _IngestJson) - except (msgspec.DecodeError, msgspec.ValidationError) as exc: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "code": "validation_failed", - "message": str(exc), + "message": f"run directory not found at {path}", }, - ) from exc - history: list[RunLogEntry] = [] - for raw in payload.history: - state = str(raw.get("state", "")) if isinstance(raw, dict) else "" - if not state: - continue - extras = ( - {k: v for k, v in raw.items() if k not in {"state", "at", "host"}} - if isinstance(raw, dict) - else {} - ) - history.append( - RunLogEntry( - state=state, - at=raw.get("at") if isinstance(raw, dict) else None, - host=raw.get("host") if isinstance(raw, dict) else None, - payload=extras, - ) ) + deps = require_deps(request) + history, current_state = await _run_log_from_queue(deps, path) return RunLogResponse( path=str(path), - current_state=str(payload.current_state) if payload.current_state else None, + current_state=current_state, history=history, ) @@ -362,6 +338,40 @@ async def get_run(run_path: str) -> RunDetail: # --------------------------------------------------------------------------- +async def _run_log_from_queue(deps: Any, run_path: Path) -> tuple[list[RunLogEntry], str | None]: + """Derive a run's per-run log from its sync-queue job. + + Returns ``(history, current_state)``. The operator-free per-file NAS + sync redesign (2026-05-21) removed the ``ingest.json`` history; the + sync queue is the run-status source this phase. A run with no job + yields an empty history and ``current_state == "none"``. + """ + nas_sync = getattr(deps, "nas_sync", None) + getter = getattr(nas_sync, "get_by_run_path", None) if nas_sync is not None else None + if getter is None: + return [], "none" + try: + row = await getter(run_path) + except Exception as exc: # pragma: no cover -- defensive + _log.warning("sync-queue lookup failed for %s: %s", run_path, exc) + return [], "none" + if row is None: + return [], "none" + state = getattr(getattr(row, "state", None), "value", None) or "none" + extras: dict[str, Any] = {} + for field in ("attempts", "verify_passes", "last_error", "nas_path"): + value = getattr(row, field, None) + if value: + extras[field] = value + entry = RunLogEntry( + state=state, + at=getattr(row, "verified_at", None) or getattr(row, "enqueued_at", None) or None, + host=None, + payload=extras, + ) + return [entry], state + + def _path_is_under_allowed_root(path: Path, config: Any) -> bool: """Return True if ``path`` is under any of the configured roots. @@ -471,33 +481,30 @@ def _relay_label_from_first_run(equipment_dir: Path, fallback: str) -> str: return fallback -def _per_file_sync_status(path: Path) -> str | None: - """Return per-file sync status for a folder-list row. - - Redesign §5 last bullet: - - For files under owned ``nas`` equipment: derived from the run's - ``creation.json`` ``sync_status`` (pending → synced → verified). - - For files under owned ``stage`` equipment: tops out at ``relayed``. - - For files under received equipment: derived from ``ingest.json`` - lifecycle state. - - Implemented conservatively: walks up to find the nearest - ``creation.json`` (run cache) and returns its ``sync_status`` if - present. Receives a ``None`` when nothing applies. +# Per-file GUI display states (operator-free per-file NAS sync design, +# 2026-05-21). The five-state table in the spec maps a file's +# sync_state.json record + on-disk presence to one of these. The UI's +# ``sync_status_props`` renders an icon for each. +FILE_STATE_ACQUIRING = "acquiring" +FILE_STATE_SYNCING = "syncing" +FILE_STATE_SYNCED = "synced" +FILE_STATE_ON_NAS = "on_nas" + + +def _find_run_root(folder: Path) -> Path | None: + """Walk up from ``folder`` to the nearest run directory, or ``None``. + + A run directory is identified by the presence of a + ``.exlab-wizard/creation.json`` cache. The bound (10) tolerates + typical instrument output tree depths without becoming pathological + for misrooted paths. """ - if path.is_dir(): - return None - # Walk up to find a Run_*/.exlab-wizard/creation.json. The bound (10) - # tolerates typical instrument output tree depths (Run/data/raw/ - # series/frames/...) without becoming pathological for misrooted paths. - current = path.parent + current = folder for _ in range(10): try: - cache_path = creation_json_path(current) - if cache_path.exists(): - payload = read_msgspec_json(cache_path, CreationJson) - return payload.sync_status - except (msgspec.DecodeError, msgspec.ValidationError, OSError): + if creation_json_path(current).exists(): + return current + except OSError: return None if current.parent == current: break @@ -505,6 +512,70 @@ def _per_file_sync_status(path: Path) -> str | None: return None +def _read_sync_state(run_root: Path) -> Any: + """Read a run's ``sync_state.json`` via :class:`SyncStateWriter`. + + Returns the decoded :class:`SyncStateJson` (an empty state when the + file is absent), or ``None`` on a decode/IO failure so callers can + fall back to a no-status row. + """ + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + try: + return SyncStateWriter().read_sync(run_root) + except Exception as exc: # pragma: no cover -- defensive + _log.warning("sync_state.json read failed for %s: %s", run_root, exc) + return None + + +def _file_state_from_record(record: Any | None, *, on_disk: bool) -> str | None: + """Map a ``sync_state.json`` record + on-disk presence to a GUI state. + + The five-state table (operator-free per-file NAS sync design, + 2026-05-21): + + * not recorded + on disk -> ``acquiring`` + * recorded, unverified + disk -> ``syncing`` + * recorded, verified + on disk -> ``synced`` + * recorded, verified + absent -> ``on_nas`` (tombstone) + """ + if record is None: + return FILE_STATE_ACQUIRING if on_disk else None + verified = getattr(record, "verified_at", None) is not None + if on_disk: + return FILE_STATE_SYNCED if verified else FILE_STATE_SYNCING + return FILE_STATE_ON_NAS if verified else None + + +def _per_file_sync_status(path: Path) -> str | None: + """Return the per-file GUI sync state for a folder-list row. + + Operator-free per-file NAS sync design (2026-05-21): per-file status + is sourced from the run's ``sync_state.json`` rather than the + ``creation.json`` ``sync_status`` field. Returns ``None`` for + directories or for files outside any run. + """ + if path.is_dir(): + return None + run_root = _find_run_root(path.parent) + if run_root is None: + return None + state = _read_sync_state(run_root) + if state is None: + return None + rel = _run_relative_posix(run_root, path) + record = state.files.get(rel) if rel is not None else None + return _file_state_from_record(record, on_disk=True) + + +def _run_relative_posix(run_root: Path, path: Path) -> str | None: + """Return ``path`` relative to ``run_root`` as a POSIX string, or ``None``.""" + try: + return path.relative_to(run_root).as_posix() + except ValueError: + return None + + def _iter_run_or_project_subdirs(parent: Path) -> list[os.DirEntry[str]]: """Return name-sorted real subdirectories of ``parent``, sans the cache. @@ -579,15 +650,19 @@ def _scan_run_children( def _build_run_node(run_dir: Path, *, kind: str) -> RunNode: + """Build a tree run node, deriving its rollup from ``sync_state.json``. + + Operator-free per-file NAS sync design (2026-05-21): the run-node + rollup (``syncing`` / ``synced`` / ``cleared``) is derived from the + run's ``sync_state.json`` via :meth:`SyncStateWriter.rollup_state`, + not the ``creation.json`` ``sync_status`` field. The rollup is + ``None`` only when the run has no ``creation.json`` at all. + """ cache_path = creation_json_path(run_dir) - sync_status: str | None = None has_cache = cache_path.exists() + sync_status: str | None = None if has_cache: - try: - payload = read_msgspec_json(cache_path, CreationJson) - sync_status = payload.sync_status - except (msgspec.DecodeError, msgspec.ValidationError): - sync_status = None + sync_status = _run_rollup_status(run_dir) return RunNode( name=run_dir.name, path=str(run_dir), @@ -597,6 +672,22 @@ def _build_run_node(run_dir: Path, *, kind: str) -> RunNode: ) +def _run_rollup_status(run_dir: Path) -> str | None: + """Return a run's derived ``RunSyncState`` rollup value, or ``None``. + + Reads ``sync_state.json`` and applies the pure + :meth:`SyncStateWriter.rollup_state` derivation. + """ + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + try: + state = SyncStateWriter().read_sync(run_dir) + except Exception as exc: # pragma: no cover -- defensive + _log.warning("sync_state.json rollup read failed for %s: %s", run_dir, exc) + return None + return SyncStateWriter.rollup_state(state).value + + # --------------------------------------------------------------------------- # Run-detail helpers # --------------------------------------------------------------------------- @@ -676,12 +767,24 @@ def scan_folder_sync(folder_path: str, config: Any) -> FolderResponse: "message": f"cannot list {path}: {exc}", }, ) from exc + # Source per-file status from the run's sync_state.json once for the + # whole folder (operator-free per-file NAS sync design, 2026-05-21). + run_root = _find_run_root(path) + sync_state = _read_sync_state(run_root) if run_root is not None else None + on_disk_rel: set[str] = set() for entry in sorted(scandir_entries, key=lambda e: e.name): try: is_dir = entry.is_dir(follow_symlinks=False) stat = entry.stat(follow_symlinks=False) except OSError: continue + rel: str | None = None + record: Any | None = None + if sync_state is not None and run_root is not None and not is_dir: + rel = _run_relative_posix(run_root, Path(entry.path)) + if rel is not None: + on_disk_rel.add(rel) + record = sync_state.files.get(rel) entries.append( FolderEntry( name=entry.name, @@ -689,12 +792,64 @@ def scan_folder_sync(folder_path: str, config: Any) -> FolderResponse: is_dir=is_dir, size_bytes=None if is_dir else stat.st_size, modified_iso=dt_to_iso(datetime.fromtimestamp(stat.st_mtime, tz=UTC)), - sync_status=_per_file_sync_status(Path(entry.path)), + sync_status=(None if is_dir else _file_state_from_record(record, on_disk=True)), + keep_local=bool(getattr(record, "keep_local", False)), ) ) + # Tombstones: files recorded in sync_state.json but absent on disk + # (a cleared run still lists its files as "On NAS"). Emitted only for + # records that resolve to this folder (the rel-path's parent matches). + if sync_state is not None and run_root is not None: + entries.extend(_tombstone_entries(run_root, path, sync_state, on_disk_rel)) + entries.sort(key=lambda e: e.name) return FolderResponse(path=str(path), entries=entries) +def _tombstone_entries( + run_root: Path, + folder: Path, + sync_state: Any, + on_disk_rel: set[str], +) -> list[FolderEntry]: + """Build "On NAS" tombstone rows for cleared-run files absent on disk. + + Operator-free per-file NAS sync design (2026-05-21): ``sync_state.json`` + survives cleanup, so a cleared run still expands to show every file as + an "On NAS" tombstone. Only records whose run-relative path resolves to + a file *directly inside* ``folder`` and that is not already on disk are + emitted. + """ + out: list[FolderEntry] = [] + for rel, record in sync_state.files.items(): + if rel in on_disk_rel: + continue + abs_path = run_root / rel + # Only keys resolving to a file directly inside ``folder`` are + # emitted. This intentionally drops any non-normalized + # sync_state.json key (a ``..`` segment or an absolute path): + # such a key's joined path won't have ``folder`` as its parent, + # so it is silently skipped. Safe -- no file is ever opened here. + if abs_path.parent != folder: + continue + state = _file_state_from_record(record, on_disk=False) + if state is None: + # An unverified, absent record is not a meaningful tombstone. + continue + out.append( + FolderEntry( + name=abs_path.name, + path=str(abs_path), + is_dir=False, + size_bytes=None, + modified_iso=None, + sync_status=state, + keep_local=bool(getattr(record, "keep_local", False)), + tombstone=True, + ) + ) + return out + + def build_hierarchy_dict(config: Any) -> dict[Any, dict[Any, list[Any]]]: """Compose the nested hierarchy dict that ``ui.components.tree.build_tree`` expects. diff --git a/src/exlab_wizard/api/routers/config.py b/src/exlab_wizard/api/routers/config.py index 91d570c..d493223 100644 --- a/src/exlab_wizard/api/routers/config.py +++ b/src/exlab_wizard/api/routers/config.py @@ -19,9 +19,10 @@ from fastapi import APIRouter, HTTPException, Request, status from pydantic import BaseModel, ConfigDict -from exlab_wizard.api._dependencies import require_deps -from exlab_wizard.config.models import Config, EquipmentConfig +from exlab_wizard.api._dependencies import lims_password_present, require_deps +from exlab_wizard.config.models import Config, EquipmentConfig, config_with_equipment_appended from exlab_wizard.constants import SetupState +from exlab_wizard.errors import ConfigError from exlab_wizard.logging import get_logger from exlab_wizard.paths import ( evaluate_setup_state, @@ -94,7 +95,7 @@ async def put_config(request: Request, body: Config) -> ConfigUpdateResponse: state = evaluate_setup_state( deps.config, lims_reachable=getattr(deps, "lims_reachable", True), - keyring_password_present=getattr(deps, "keyring_password_present", True), + keyring_password_present=lims_password_present(deps), ) return ConfigUpdateResponse( state=state.value, @@ -114,21 +115,13 @@ async def append_equipment(request: Request, body: EquipmentConfig) -> Equipment Duplicate IDs are rejected with a structured error per §10. """ deps = require_deps(request) - config = getattr(deps, "config", None) or Config() - for entry in config.equipment: - if entry.id == body.id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "code": "equipment_id_conflict", - "message": (f"equipment id {body.id!r} already exists in config"), - }, - ) - new_equipment = [*config.equipment, body] - # model_validate re-runs the cross-field invariants (unique-id check - # etc.) on the merged config. - new_config = config.model_copy(update={"equipment": new_equipment}) - Config.model_validate(new_config.model_dump(mode="python")) + try: + new_config = config_with_equipment_appended(getattr(deps, "config", None), body) + except ConfigError as exc: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "equipment_id_conflict", "message": str(exc)}, + ) from exc saver = getattr(deps, "save_config", None) if saver is not None: await _await_or_call(saver, new_config) @@ -136,7 +129,7 @@ async def append_equipment(request: Request, body: EquipmentConfig) -> Equipment state = evaluate_setup_state( deps.config, lims_reachable=getattr(deps, "lims_reachable", True), - keyring_password_present=getattr(deps, "keyring_password_present", True), + keyring_password_present=lims_password_present(deps), ) return EquipmentAppendResponse( appended_id=body.id, diff --git a/src/exlab_wizard/api/routers/staging.py b/src/exlab_wizard/api/routers/staging.py index e38d046..5160549 100644 --- a/src/exlab_wizard/api/routers/staging.py +++ b/src/exlab_wizard/api/routers/staging.py @@ -27,21 +27,22 @@ from pydantic import BaseModel, ConfigDict from exlab_wizard.api._dependencies import require_deps -from exlab_wizard.cache.ingest_writer import IngestWriter from exlab_wizard.config.models import Config -from exlab_wizard.constants import IngestState, SyncHandleState +from exlab_wizard.constants import RunSyncState, SyncHandleState from exlab_wizard.logging import get_logger +from exlab_wizard.orchestrator.staging_clear import clear_run_dir from exlab_wizard.orchestrator.staging_query import ( StagedRunSummary, list_staged_runs, ) -from exlab_wizard.paths import ingest_json_path from exlab_wizard.utils.time import utc_now __all__ = [ "ClearResponse", "ClearVerifiedResponse", "ForceSyncResponse", + "KeepLocalRequest", + "KeepLocalResponse", "StagedRunRow", "StagingListResponse", "build_staging_router", @@ -107,6 +108,31 @@ class ClearVerifiedResponse(BaseModel): cleared_paths: list[str] +class KeepLocalRequest(BaseModel): + """``POST /staging/{run_path}/keep-local`` request body. + + Operator-free per-file NAS sync design (2026-05-21): the operator + toggles a file's ``keep_local`` flag from the file-list context menu. + ``relative_path`` is the run-relative POSIX path of the file within + the run directory; ``keep_local`` is the desired flag value. + """ + + model_config = ConfigDict(extra="forbid") + + relative_path: str + keep_local: bool + + +class KeepLocalResponse(BaseModel): + """``POST /staging/{run_path}/keep-local`` response.""" + + model_config = ConfigDict(extra="forbid") + + run_path: str + relative_path: str + keep_local: bool + + # --------------------------------------------------------------------------- # Router builder # --------------------------------------------------------------------------- @@ -120,7 +146,11 @@ def build_staging_router() -> APIRouter: async def get_staging(request: Request) -> StagingListResponse: deps = require_deps(request) config = _require_config(deps) - rows = list_staged_runs(config=config, now_utc=utc_now()) + rows = list_staged_runs( + config=config, + now_utc=utc_now(), + sync_state_writer=getattr(deps, "sync_state_writer", None), + ) return StagingListResponse(runs=[_row_from_summary(s) for s in rows]) @router.post( @@ -128,29 +158,92 @@ async def get_staging(request: Request) -> StagingListResponse: response_model=ClearVerifiedResponse, ) async def post_clear_verified(request: Request) -> ClearVerifiedResponse: - """Bulk-clear every staged run in ``sync_verified`` state. + """Bulk-clear every staged run whose NAS sync is verified. Redesign §4.6: the file-explorer footer's "Clear verified runs" - action. Routes through the same - :func:`exlab_wizard.orchestrator.cleanup.clear_run` primitive - as the per-run endpoint, so failure modes (missing dirs, ingest - write errors) behave identically. Returns the list of cleared - run paths so the UI can report a count. + action. Phase 5 keys "clearable" off the ``sync_state.json`` + ``SYNCED`` rollup -- a run is clearable when every tracked file is + verified on the NAS and the run has not already been cleared. """ deps = require_deps(request) config = _require_config(deps) - ingest_writer = _require_ingest_writer(deps) - # Deferred import: see the per-run /clear endpoint below for the - # cycle-avoidance rationale. - from exlab_wizard.orchestrator.cleanup import clear_all_verified - - cleared = await clear_all_verified( + cleared: list[str] = [] + for summary in list_staged_runs( config=config, - ingest_writer=ingest_writer, - ) + sync_state_writer=getattr(deps, "sync_state_writer", None), + ): + # Only a fully-SYNCED run is clearable; ``cleared`` runs have + # no staging copy left and ``syncing`` runs are unproven. + if summary.current_state != RunSyncState.SYNCED.value: + continue + run_path = Path(summary.path) + try: + files, _bytes = clear_run_dir(run_path) + except Exception as exc: + _log.warning("clear-verified: clear failed for %s: %s", run_path, exc) + continue + if files > 0: + cleared.append(str(run_path)) _log.info("clear-verified bulk action: cleared=%d", len(cleared)) return ClearVerifiedResponse(cleared_paths=cleared) + @router.post( + "/staging/{run_path:path}/keep-local", + response_model=KeepLocalResponse, + ) + async def post_keep_local( + request: Request, + run_path: str, + body: KeepLocalRequest, + ) -> KeepLocalResponse: + """Toggle a file's ``keep_local`` flag in the run's ``sync_state.json``. + + Operator-free per-file NAS sync design (2026-05-21): a + ``keep_local`` file still syncs to the NAS but is excluded from + cleanup deletion. ``sync_state.json`` has a single writer -- the + orchestrator's :class:`SyncStateWriter` -- so the GUI never writes + the file directly; it calls this endpoint instead. Returns 503 + when no ``SyncStateWriter`` is wired on the app instance, and 404 + when ``run_path`` is not a real run inside an allowed root. + """ + deps = require_deps(request) + config = _require_config(deps) + writer = getattr(deps, "sync_state_writer", None) + if writer is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={ + "code": "sync_state_writer_unavailable", + "message": "sync-state writer is not wired on this app instance", + }, + ) + # Path-containment guard: ``run_path`` comes straight from the URL + # and ``SyncStateWriter`` *creates* ``/.exlab-wizard/ + # sync_state.json``. Reject any path that is not a real run -- it + # must sit under an allowed root and carry a ``creation.json`` + # cache -- so a hostile path (``%2Fetc``) cannot provoke a write. + path = Path(run_path) + if not _is_real_run(path, config): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": "session_not_found", + "message": f"no run found at {run_path}", + }, + ) + await writer.set_keep_local(path, body.relative_path, body.keep_local) + _log.info( + "keep-local toggled via API: run=%s file=%s value=%s", + run_path, + body.relative_path, + body.keep_local, + ) + return KeepLocalResponse( + run_path=run_path, + relative_path=body.relative_path, + keep_local=body.keep_local, + ) + @router.post( "/staging/{run_path:path}/force-sync", response_model=ForceSyncResponse, @@ -183,40 +276,28 @@ async def post_force_sync(request: Request, run_path: str) -> ForceSyncResponse: async def post_clear(request: Request, run_path: str) -> ClearResponse: deps = require_deps(request) config = _require_config(deps) - ingest_writer = _require_ingest_writer(deps) path = Path(run_path) - # Defensive check: the spec only allows clearing sync-verified - # runs (manual mode). The watcher would never call this on - # earlier states, but the API is operator-facing so we enforce - # the rule here too. - ingest_path = ingest_json_path(path) - if ingest_path.exists(): - try: - payload = await ingest_writer.read_ingest(ingest_path) - except Exception: - payload = None - if payload is not None and payload.current_state != IngestState.SYNC_VERIFIED: + # Defensive check: only a fully-SYNCED run may be cleared. Phase 5 + # derives the run rollup from ``sync_state.json``; a ``syncing`` run + # is unproven and a ``cleared`` run has no staging copy left. + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + writer = getattr(deps, "sync_state_writer", None) + if writer is not None: + state = SyncStateWriter.rollup_state(writer.read_sync(path)).value + if state != RunSyncState.SYNCED.value: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ "code": "staging_not_sync_verified", "message": ( - f"Cannot clear run in state {payload.current_state!r}; " - "only sync_verified runs may be cleared." + f"Cannot clear run in sync state {state!r}; " + "only synced runs may be cleared." ), }, ) - # Deferred import: ``orchestrator.cleanup`` pulls in - # ``api.schemas`` -> ``api`` package, so a module-level import - # here creates an ``api.routers.staging`` <-> ``orchestrator`` - # cycle whenever ``orchestrator`` is imported before ``api``. - from exlab_wizard.orchestrator.cleanup import clear_run - - files_freed, bytes_freed = await clear_run( - path, - config=config, - ingest_writer=ingest_writer, - ) + _ = config # kept for parity / future hooks + files_freed, bytes_freed = clear_run_dir(path) return ClearResponse( run_path=run_path, files_freed=files_freed, @@ -245,6 +326,29 @@ def _row_from_summary(summary: StagedRunSummary) -> StagedRunRow: ) +def _is_real_run(run_path: Path, config: Config) -> bool: + """Return True when ``run_path`` is a real run inside an allowed root. + + A "real run" is a directory that (a) sits under one of the configured + roots (local_root / staging_root / templates / plugins) -- reusing the + same containment helper the ``GET /folder`` endpoint uses -- and (b) + carries a ``.exlab-wizard/creation.json`` cache. The containment check + guards against a hostile URL path; the ``creation.json`` check ensures + ``SyncStateWriter`` only ever creates ``sync_state.json`` under a + genuine run directory. + """ + from exlab_wizard.api.routers.browse import _path_is_under_allowed_root + from exlab_wizard.paths import creation_json_path + + try: + resolved = run_path.resolve() + except OSError: + return False + if not _path_is_under_allowed_root(resolved, config): + return False + return creation_json_path(run_path).exists() + + def _require_config(deps: Any) -> Config: """Return the live :class:`Config` or raise 503 when no config is wired. @@ -274,12 +378,3 @@ def _require_nas_sync(deps: Any) -> Any: }, ) return nas_sync - - -def _require_ingest_writer(deps: Any) -> IngestWriter: - writer = getattr(deps, "ingest_writer", None) - if writer is None: - # Fall back to a freshly constructed writer; the IngestWriter is - # stateless across calls (one FileLock per ingest path). - return IngestWriter() - return writer diff --git a/src/exlab_wizard/api/schemas.py b/src/exlab_wizard/api/schemas.py index be3879d..df1c0c7 100644 --- a/src/exlab_wizard/api/schemas.py +++ b/src/exlab_wizard/api/schemas.py @@ -42,12 +42,10 @@ from msgspec import json as msgspec_json from msgspec import structs as msgspec_structs +from exlab_wizard.cache.sync_state_schema import FileSyncRecord, SyncStateJson from exlab_wizard.constants import ( - CompletenessSignal, CreationLevel, - IngestState, LIMSProjectSource, - OrchestratorTransportType, PluginStatus, RunKind, RunScope, @@ -57,7 +55,7 @@ __all__ = [ "CreationJson", "EquipmentJson", - "IngestJson", + "FileSyncRecord", "LimsProjectBlock", "OrchestratorBlock", "OverrideEntry", @@ -65,6 +63,7 @@ "PluginApplied", "PluginIsolation", "ReadmeFieldsJson", + "SyncStateJson", "TemplateBlock", "TestRunsJson", "TombstoneEntry", @@ -169,21 +168,18 @@ class OrchestratorBlock( role and the orchestrator block always carries the producing device's identity. - The ``equipment_label`` / ``completeness_signal`` / - ``sentinel_filename`` / ``manifest_filename`` fields travel with the - push so the receiving orchestrator can auto-discover received - equipment (Redesign Spec §3.3) without a per-equipment registry of - its own. They are optional for forward-compat with creation.json - files written by earlier writers. + The ``equipment_label`` field travels with the push so the receiving + orchestrator can auto-discover received equipment (Redesign Spec §3.3) + without a per-equipment registry of its own. It is optional for + forward-compat with creation.json files written by earlier writers. + The former completeness-signal relay fields were removed by the + operator-free quiescence-sync redesign (2026-05-21). """ enabled: bool host: str label: str equipment_label: str | None = None - completeness_signal: CompletenessSignal | None = None - sentinel_filename: str | None = None - manifest_filename: str | None = None # --------------------------------------------------------------------------- @@ -371,33 +367,14 @@ class TestRunsJson( # --------------------------------------------------------------------------- -# ingest.json (§13.4) -- orchestrator-only staging lifecycle record +# sync_state.json -- per-run, per-file quiescence-driven sync state +# (operator-free per-file NAS sync design, 2026-05-21). +# +# The ``SyncStateJson`` / ``FileSyncRecord`` structs are defined in the +# dependency-free ``exlab_wizard.cache.sync_state_schema`` module so the +# ``cache`` / ``sync`` / ``orchestrator`` packages can import them (and the +# ``SyncStateWriter``) at module scope without forming a circular import +# back through the ``api`` package. They are re-exported here so the +# "single source of truth for cache schemas" surface stays intact for API +# callers. # --------------------------------------------------------------------------- - - -class IngestJson( - Struct, - omit_defaults=True, - forbid_unknown_fields=False, -): - """``ingest.json`` orchestrator staging record at schema version 1.1. Spec §13.4. - - Written by the orchestrator only (not by equipment workstations). The - ``history`` list is append-only per §13: lifecycle transitions are - recorded, never overwritten. ``current_state`` mirrors the most recent - history entry's ``state`` for fast read-without-walk access. - - History entries are loose dicts because the optional fields per state - (``files_received`` / ``bytes_received`` on ``complete``; ``nas_path`` / - ``checksum_file`` on ``sync_verified``) make a strict type a nuisance. - The state-machine validation is performed by the writer. - """ - - schema_version: str - project_name: str - equipment_id: str - run_kind: RunKind - run_path: str - transport: OrchestratorTransportType - current_state: IngestState - history: list[dict[str, Any]] = [] diff --git a/src/exlab_wizard/api/setup.py b/src/exlab_wizard/api/setup.py index d0e962b..5663202 100644 --- a/src/exlab_wizard/api/setup.py +++ b/src/exlab_wizard/api/setup.py @@ -28,7 +28,7 @@ from fastapi import APIRouter, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field -from exlab_wizard.api._dependencies import require_deps +from exlab_wizard.api._dependencies import lims_password_present, require_deps from exlab_wizard.config.models import ( EquipmentConfig, EquipmentTransport, @@ -151,7 +151,7 @@ def compute_setup_state(deps: Any) -> SetupState: return evaluate_setup_state( deps.config, lims_reachable=getattr(deps, "lims_reachable", True), - keyring_password_present=getattr(deps, "keyring_password_present", True), + keyring_password_present=lims_password_present(deps), ) diff --git a/src/exlab_wizard/cache/ingest_writer.py b/src/exlab_wizard/cache/ingest_writer.py deleted file mode 100644 index 904232b..0000000 --- a/src/exlab_wizard/cache/ingest_writer.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Orchestrator-only writer for ``ingest.json``. Backend Spec §13.4, §4.4.5. - -The orchestrator writes one ``ingest.json`` per staged run, capturing the -five-state lifecycle (§13.3): ``staging`` -> ``complete`` -> ``sync_queued`` --> ``sync_verified`` -> ``cleared``. State transitions are append-only -- a -new entry is added to the ``history`` array on every transition, and the -top-level ``current_state`` mirrors the latest entry. - -Disk-side guarantees per §4.4.5: - -* ``msgspec.json`` for typed encode/decode (schema validation in one pass). -* ``filelock.FileLock`` advisory exclusive lock around the read-mutate-write - cycle so concurrent appends never lose entries. -* Atomic write via tempfile + ``fsync`` + ``os.replace``. - -Reads enforce the §11.9.2 reader policy: a file at a different schema major -than the writer raises ``SchemaMajorMismatchError`` (no silent partial-parse -across major boundaries). -""" - -from __future__ import annotations - -import asyncio -import socket -from pathlib import Path -from typing import Any - -import msgspec -from filelock import FileLock - -from exlab_wizard.api.schemas import IngestJson -from exlab_wizard.cache import lock_path_for -from exlab_wizard.constants import INGEST_JSON_VERSION, IngestState -from exlab_wizard.io import atomic_write_bytes, read_msgspec_json -from exlab_wizard.logging import get_logger -from exlab_wizard.utils.state import assert_forward_transition -from exlab_wizard.utils.time import utc_now_iso - -__all__ = ["IngestWriter"] - -_logger = get_logger(__name__) - -# Forward state-machine map per §13.3. Each key is the source state; the -# value is the set of states reachable in one step. Any transition not -# present here is rejected by ``append_state_transition``. -_FORWARD_TRANSITIONS: dict[IngestState, frozenset[IngestState]] = { - IngestState.STAGING: frozenset({IngestState.COMPLETE}), - IngestState.COMPLETE: frozenset({IngestState.SYNC_QUEUED}), - IngestState.SYNC_QUEUED: frozenset({IngestState.SYNC_VERIFIED}), - IngestState.SYNC_VERIFIED: frozenset({IngestState.CLEARED}), - IngestState.CLEARED: frozenset(), -} - -# Reader's expected major version (every writer always emits this major). -_EXPECTED_MAJOR: int = int(INGEST_JSON_VERSION.split(".", 1)[0]) - - -class IngestWriter: - """Writer for ``ingest.json``. Orchestrator-mode only. - - Backend Spec §13.4. State history is append-only (§13.3) to preserve full - audit history -- the writer never edits or deletes prior entries. - - All public methods are ``async`` to match the §4.4.5 ``CacheWriter`` - contract; the blocking lock + I/O work is dispatched through - ``asyncio.to_thread`` so the FastAPI event loop is never blocked. - """ - - async def write_ingest(self, path: Path, payload: IngestJson) -> None: - """Write ``payload`` to ``path`` atomically under an exclusive lock. - - Reserved for the initial-creation path (no file exists yet); the - per-file lock is taken defensively so a concurrent initial-write - attempt serializes rather than races. - """ - await asyncio.to_thread(self._write_ingest_blocking, path, payload) - - async def read_ingest(self, path: Path) -> IngestJson: - """Read and decode ``path`` into an ``IngestJson``. - - Raises ``SchemaMajorMismatchError`` (§11.9.2) when the on-disk file - carries a different schema major than ``INGEST_JSON_VERSION``. - """ - return await asyncio.to_thread(self._read_ingest_blocking, path) - - async def append_state_transition( - self, - path: Path, - new_state: IngestState, - *, - host: str, - files_received: int | None = None, - bytes_received: int | None = None, - nas_path: str | None = None, - checksum_file: str | None = None, - ) -> IngestJson: - """Append a state-transition entry and update ``current_state``. - - File-locked for the entire read-mutate-write cycle so concurrent - callers never lose entries. The new history entry has the shape:: - - {"state": "", "at": "", "host": ""} - - Per §13.4 the entry carries optional extras when transitioning to - specific states: - - * ``complete`` -- ``files_received`` and ``bytes_received``. - * ``sync_verified`` -- ``nas_path`` and ``checksum_file``. - - Other state transitions ignore those extras (they are silently dropped - because the spec does not define their meaning at those states). - - Raises ``ValueError`` if ``new_state`` is not a permitted forward - transition from the file's current state. Going backward (e.g. - ``cleared`` -> ``staging``) is rejected. The full state machine is - documented in ``_FORWARD_TRANSITIONS`` above and §13.3. - """ - return await asyncio.to_thread( - self._append_state_transition_blocking, - path, - new_state, - host, - files_received, - bytes_received, - nas_path, - checksum_file, - ) - - # ---- Blocking helpers (run via asyncio.to_thread) --------------------- - - def _write_ingest_blocking(self, path: Path, payload: IngestJson) -> None: - with FileLock(lock_path_for(path)): - atomic_write_bytes(path, msgspec.json.encode(payload)) - _logger.info( - "ingest.json written: %s (current_state=%s)", - path, - payload.current_state, - ) - - def _read_ingest_blocking(self, path: Path) -> IngestJson: - with FileLock(lock_path_for(path)): - return self._decode_ingest_locked(path) - - @staticmethod - def _decode_ingest_locked(path: Path) -> IngestJson: - """Decode ``ingest.json`` with the §11.9.2 schema-major gate. - - Caller MUST already hold the per-file ``FileLock``. - """ - return read_msgspec_json(path, IngestJson, expected_major=_EXPECTED_MAJOR) - - def _append_state_transition_blocking( - self, - path: Path, - new_state: IngestState, - host: str, - files_received: int | None, - bytes_received: int | None, - nas_path: str | None, - checksum_file: str | None, - ) -> IngestJson: - with FileLock(lock_path_for(path)): - payload = self._decode_ingest_locked(path) - - current = IngestState(payload.current_state) - assert_forward_transition(current, new_state, _FORWARD_TRANSITIONS) - - entry: dict[str, Any] = { - "state": new_state.value, - "at": utc_now_iso(), - "host": host, - } - if new_state is IngestState.COMPLETE: - if files_received is not None: - entry["files_received"] = files_received - if bytes_received is not None: - entry["bytes_received"] = bytes_received - elif new_state is IngestState.SYNC_VERIFIED: - if nas_path is not None: - entry["nas_path"] = nas_path - if checksum_file is not None: - entry["checksum_file"] = checksum_file - - new_history = [*payload.history, entry] - new_payload = msgspec.structs.replace( - payload, - current_state=new_state.value, - history=new_history, - ) - atomic_write_bytes(path, msgspec.json.encode(new_payload)) - - _logger.info( - "ingest.json transition: %s -> %s (host=%s, path=%s)", - current.value, - new_state.value, - host, - path, - ) - return new_payload - - -# Convenience for callers that need a default host string. Not part of the -# public class -- exposed for tests and the orchestrator session bootstrap. -def default_host() -> str: - """Return ``socket.gethostname()`` (orchestrator default for ``host``).""" - return socket.gethostname() diff --git a/src/exlab_wizard/cache/sync_state_schema.py b/src/exlab_wizard/cache/sync_state_schema.py new file mode 100644 index 0000000..eb29054 --- /dev/null +++ b/src/exlab_wizard/cache/sync_state_schema.py @@ -0,0 +1,62 @@ +"""``msgspec.Struct`` types for the per-run ``sync_state.json`` cache. + +Operator-free per-file NAS sync design (2026-05-21). These structs live in +the dependency-free ``cache`` package -- not ``api.schemas`` -- so the +``cache``, ``sync``, and ``orchestrator`` modules can import them (and +:class:`~exlab_wizard.cache.sync_state_writer.SyncStateWriter`) at module +scope without re-entering the ``api`` package and forming a circular +import. ``api.schemas`` re-exports both names so the wider "single source +of truth for cache schemas" surface is preserved for API callers. +""" + +from __future__ import annotations + +from msgspec import Struct + +__all__ = ["FileSyncRecord", "SyncStateJson"] + + +class FileSyncRecord( + Struct, + omit_defaults=True, + forbid_unknown_fields=False, +): + """Per-file sync record stored under ``sync_state.json``'s ``files`` map. + + One record per run-relative POSIX path. These records are *freely + mutated in place* by ``SyncStateWriter`` as a file is synced, + re-modified, and re-synced. + + * ``synced_signature`` -- the ``(st_size, st_mtime_ns)`` captured at the + last successful sync. ``None`` means the file has never synced. A file + whose current signature differs is "modified since sync" and becomes + eligible again once it re-settles. + * ``verified_at`` -- the ISO-8601 timestamp at which the file's SHA-256 + was confirmed against the NAS copy. ``None`` until verified; a file + counts toward the ``SYNCED`` rollup only once this is set. + * ``keep_local`` -- when ``True`` the file still syncs to the NAS but is + excluded from cleanup deletion. + """ + + synced_signature: tuple[int, int] | None = None + verified_at: str | None = None + keep_local: bool = False + + +class SyncStateJson( + Struct, + omit_defaults=True, + forbid_unknown_fields=False, +): + """``sync_state.json`` per-run sync-state record at schema version 1.0. + + Written by the orchestrator only. A *freely-mutable current-state map* + (not append-only): ``files`` maps a run-relative POSIX path to its + :class:`FileSyncRecord`. ``cleared_at`` is set once the run's staging + copy has been cleaned up. The run-level ``SYNCING``/``SYNCED``/``CLEARED`` + rollup is derived on read from this payload, never persisted as such. + """ + + schema_version: str + cleared_at: str | None = None + files: dict[str, FileSyncRecord] = {} diff --git a/src/exlab_wizard/cache/sync_state_writer.py b/src/exlab_wizard/cache/sync_state_writer.py new file mode 100644 index 0000000..9cb0e9d --- /dev/null +++ b/src/exlab_wizard/cache/sync_state_writer.py @@ -0,0 +1,272 @@ +"""Orchestrator-only writer for ``sync_state.json``. + +Operator-free per-file NAS sync design (2026-05-21). The orchestrator +writes one ``sync_state.json`` per run pending NAS sync, under +``/.exlab-wizard/sync_state.json``. It is a **freely-mutable +current-state map**: per-file records are overwritten in place as a file +is synced, re-modified, and re-synced, and ``cleared_at`` is stamped once +when the run's staging copy is cleaned up. + +Disk-side guarantees follow the §4.4.5 ``CacheWriter`` contract: + +* ``msgspec.json`` for typed encode/decode (schema validation in one pass). +* ``filelock.FileLock`` advisory exclusive lock around every + read-mutate-write cycle so concurrent updates never lose a record. +* Atomic write via tempfile + ``fsync`` + ``os.replace`` + (:func:`~exlab_wizard.io.atomic_write_bytes`). + +The run-level ``SYNCING`` / ``SYNCED`` / ``CLEARED`` rollup is **derived on +read** by :meth:`SyncStateWriter.rollup_state` -- it is never persisted, +because it can oscillate (a ``SYNCED`` run whose file is modified again +returns to ``SYNCING``). +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import msgspec +from filelock import FileLock + +from exlab_wizard.cache import lock_path_for +from exlab_wizard.cache.sync_state_schema import FileSyncRecord, SyncStateJson +from exlab_wizard.constants import ( + SYNC_STATE_FILENAME, + SYNC_STATE_JSON_VERSION, + RunSyncState, +) +from exlab_wizard.io import atomic_write_bytes, read_msgspec_json +from exlab_wizard.logging import get_logger +from exlab_wizard.paths import cache_dir +from exlab_wizard.utils.time import utc_now_iso + +__all__ = ["SyncStateWriter"] + +_logger = get_logger(__name__) + +# Reader's expected major version (every writer always emits this major). +_EXPECTED_MAJOR: int = int(SYNC_STATE_JSON_VERSION.split(".", 1)[0]) + + +def _sync_state_path(run_path: Path) -> Path: + """Return the ``sync_state.json`` path under a run directory.""" + return cache_dir(run_path) / SYNC_STATE_FILENAME + + +def _ensure_cache_dir(path: Path) -> None: + """Create ``path``'s parent (the run's ``.exlab-wizard/`` cache) dir. + + Neither :func:`atomic_write_bytes` nor :class:`filelock.FileLock` + creates the parent directory, so a mutator targeting a run that has + no ``.exlab-wizard/`` dir yet would raise (and surface as an HTTP + 500). Every blocking mutator calls this first so a missing cache dir + can never fault the write. Idempotent. + """ + path.parent.mkdir(parents=True, exist_ok=True) + + +def _empty_state() -> SyncStateJson: + """Return a fresh, file-less :class:`SyncStateJson` at the current version.""" + return SyncStateJson(schema_version=SYNC_STATE_JSON_VERSION) + + +class SyncStateWriter: + """Writer for ``sync_state.json``. Orchestrator-mode only. + + All public methods are ``async`` to match the §4.4.5 ``CacheWriter`` + contract; the blocking lock + I/O work is dispatched through + ``asyncio.to_thread`` so the FastAPI event loop is never blocked. + """ + + async def read(self, run_path: Path) -> SyncStateJson: + """Read and decode the run's ``sync_state.json``. + + Returns a fresh empty :class:`SyncStateJson` when the file does not + exist yet -- a run with no sync activity simply has no record. Raises + ``SchemaMajorMismatchError`` (§11.9.2) when the on-disk file carries + a different schema major than ``SYNC_STATE_JSON_VERSION``. + """ + return await asyncio.to_thread(self._read_blocking, run_path) + + def read_sync(self, run_path: Path) -> SyncStateJson: + """Blocking variant of :meth:`read` for synchronous callers. + + :meth:`read` dispatches the blocking lock + decode through + ``asyncio.to_thread``. This variant runs the lock + decode inline; + synchronous read-side code -- notably + :func:`exlab_wizard.orchestrator.staging_query.list_staged_runs` and + :func:`exlab_wizard.orchestrator.staging_clear.clear_run_dir` -- calls + it directly. Some of those call sites run inside a ``@ui.page`` + handler (i.e. on the event loop); the lock + a single small JSON + decode is brief enough not to matter there, and the read-side query + is itself synchronous, so there is no ``to_thread`` hop to make. + Returns a fresh empty :class:`SyncStateJson` when the file is absent. + """ + return self._read_blocking(run_path) + + async def upsert_file( + self, + run_path: Path, + rel_path: str, + *, + synced_signature: tuple[int, int] | None = None, + verified_at: str | None = None, + ) -> SyncStateJson: + """Create or update one file record under an exclusive lock. + + The record for ``rel_path`` (a run-relative POSIX path) is created if + absent, otherwise updated in place. ``synced_signature`` and + ``verified_at`` overwrite the record's fields; the record's other + fields (notably ``keep_local``) are preserved. + """ + return await asyncio.to_thread( + self._upsert_file_blocking, + run_path, + rel_path, + synced_signature, + verified_at, + ) + + async def set_keep_local( + self, + run_path: Path, + rel_path: str, + value: bool, + ) -> SyncStateJson: + """Toggle a file's ``keep_local`` flag, creating the record if absent.""" + return await asyncio.to_thread( + self._set_keep_local_blocking, + run_path, + rel_path, + value, + ) + + async def mark_cleared(self, run_path: Path) -> SyncStateJson: + """Stamp ``cleared_at`` with the current UTC time. + + Called once the run's staging copy has been cleaned up; this flips + the derived rollup to ``CLEARED``. + """ + return await asyncio.to_thread(self._mark_cleared_blocking, run_path) + + def mark_cleared_sync(self, run_path: Path) -> SyncStateJson: + """Blocking variant of :meth:`mark_cleared` for synchronous callers. + + Used by the synchronous :func:`clear_run_dir` operator-clear path so + an operator "Clear" stamps ``cleared_at`` exactly like the automatic + cleanup reaper does. No-op-safe when ``sync_state.json`` is absent -- + a record is created carrying only ``cleared_at``. + """ + return self._mark_cleared_blocking(run_path) + + @staticmethod + def rollup_state(state: SyncStateJson) -> RunSyncState: + """Derive the run-level :class:`RunSyncState` rollup from ``state``. + + Pure function -- no I/O, no mutation. The rollup is computed on read + rather than persisted because the ``SYNCING``/``SYNCED`` distinction + oscillates as files are re-modified. + + * ``CLEARED`` -- ``cleared_at`` is set (takes precedence even if some + files are unverified, e.g. ``keep_local`` files left on disk). + * ``SYNCED`` -- ``files`` is non-empty and every record has a non-null + ``verified_at``. + * ``SYNCING`` -- otherwise (no files tracked yet, or at least one file + still unverified). + """ + if state.cleared_at is not None: + return RunSyncState.CLEARED + if state.files and all(rec.verified_at is not None for rec in state.files.values()): + return RunSyncState.SYNCED + return RunSyncState.SYNCING + + # ---- Blocking helpers (run via asyncio.to_thread) --------------------- + + def _read_blocking(self, run_path: Path) -> SyncStateJson: + path = _sync_state_path(run_path) + with FileLock(lock_path_for(path)): + return self._decode_locked(path) + + @staticmethod + def _decode_locked(path: Path) -> SyncStateJson: + """Decode ``sync_state.json``, returning an empty state if absent. + + Caller MUST already hold the per-file ``FileLock``. + """ + if not path.exists(): + return _empty_state() + return read_msgspec_json(path, SyncStateJson, expected_major=_EXPECTED_MAJOR) + + def _upsert_file_blocking( + self, + run_path: Path, + rel_path: str, + synced_signature: tuple[int, int] | None, + verified_at: str | None, + ) -> SyncStateJson: + path = _sync_state_path(run_path) + _ensure_cache_dir(path) + with FileLock(lock_path_for(path)): + payload = self._decode_locked(path) + existing = payload.files.get(rel_path, FileSyncRecord()) + record = msgspec.structs.replace( + existing, + synced_signature=synced_signature, + verified_at=verified_at, + ) + new_payload = self._with_file(payload, rel_path, record) + atomic_write_bytes(path, msgspec.json.encode(new_payload)) + _logger.info( + "sync_state.json upsert: %s (file=%s, verified=%s)", + path, + rel_path, + verified_at is not None, + ) + return new_payload + + def _set_keep_local_blocking( + self, + run_path: Path, + rel_path: str, + value: bool, + ) -> SyncStateJson: + path = _sync_state_path(run_path) + _ensure_cache_dir(path) + with FileLock(lock_path_for(path)): + payload = self._decode_locked(path) + existing = payload.files.get(rel_path, FileSyncRecord()) + record = msgspec.structs.replace(existing, keep_local=value) + new_payload = self._with_file(payload, rel_path, record) + atomic_write_bytes(path, msgspec.json.encode(new_payload)) + _logger.info( + "sync_state.json keep_local: %s (file=%s, value=%s)", + path, + rel_path, + value, + ) + return new_payload + + def _mark_cleared_blocking(self, run_path: Path) -> SyncStateJson: + path = _sync_state_path(run_path) + _ensure_cache_dir(path) + with FileLock(lock_path_for(path)): + payload = self._decode_locked(path) + new_payload = msgspec.structs.replace(payload, cleared_at=utc_now_iso()) + atomic_write_bytes(path, msgspec.json.encode(new_payload)) + _logger.info("sync_state.json marked cleared: %s", path) + return new_payload + + @staticmethod + def _with_file( + payload: SyncStateJson, + rel_path: str, + record: FileSyncRecord, + ) -> SyncStateJson: + """Return a copy of ``payload`` with ``files[rel_path]`` set to ``record``. + + Builds a fresh ``files`` mapping so the input payload is never + mutated in place. + """ + new_files = {**payload.files, rel_path: record} + return msgspec.structs.replace(payload, files=new_files) diff --git a/src/exlab_wizard/config/models.py b/src/exlab_wizard/config/models.py index 30dbf2d..940c6b4 100644 --- a/src/exlab_wizard/config/models.py +++ b/src/exlab_wizard/config/models.py @@ -35,7 +35,6 @@ from exlab_wizard.constants import ( TEMPLATE_QUESTION_ID_PATTERN, BandwidthDay, - CompletenessSignal, FieldType, OrchestratorTransportType, StagingCleanupMode, @@ -309,18 +308,10 @@ class EquipmentConfig(BaseModel): label: str = Field(min_length=1) local_root: str = Field(min_length=1) nas_root: str = Field(min_length=1) - completeness_signal: CompletenessSignal - sentinel_filename: str | None = None - manifest_filename: str | None = None sync_mode: SyncMode = SyncMode.NAS transport: EquipmentTransport | None = None orchestrator_staging_transport: OrchestratorStagingTransport | None = None - @field_serializer("completeness_signal") - def _serialize_completeness_signal(self, value: CompletenessSignal) -> str: - # Emit the bare string so YAML/JSON dumps round-trip the wire format. - return value.value - @field_serializer("sync_mode") def _serialize_sync_mode(self, value: SyncMode) -> str: return value.value @@ -337,25 +328,6 @@ def _validate_equipment_id(cls, value: str) -> str: except ConfigError as exc: raise ValueError(str(exc)) from exc - @model_validator(mode="after") - def _completeness_signal_requires_matching_filename(self) -> EquipmentConfig: - match self.completeness_signal: - case CompletenessSignal.SENTINEL_FILE: - if not self.sentinel_filename: - msg = ( - "equipment.completeness_signal == 'sentinel_file' " - "requires a non-empty sentinel_filename" - ) - raise ValueError(msg) - case CompletenessSignal.MANIFEST: - if not self.manifest_filename: - msg = ( - "equipment.completeness_signal == 'manifest' " - "requires a non-empty manifest_filename" - ) - raise ValueError(msg) - return self - @model_validator(mode="after") def _sync_mode_dictates_transport(self) -> EquipmentConfig: match self.sync_mode: @@ -504,13 +476,20 @@ class PluginsConfig(BaseModel): # --------------------------------------------------------------------------- +def _default_ignore_globs() -> list[str]: + return ["*.partial", "*.tmp"] + + class SyncConfig(BaseModel): - """``sync:`` block. NAS sync engine kill-switch + retry policy.""" + """``sync:`` block. NAS sync engine kill-switch + retry / quiescence policy.""" model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) enabled: bool = True retry_attempts: int = Field(default=3, ge=0) + quiescence_minutes: int = Field(default=10, ge=1) + ignore_globs: list[str] = Field(default_factory=_default_ignore_globs) + poll_interval_seconds: int = Field(default=120, ge=1) # --------------------------------------------------------------------------- @@ -594,3 +573,28 @@ def _cross_field_invariants(self) -> Config: # for completion. Pydantic validation only ensures the fields are # present (which they always are due to the empty-string defaults). return self + + +def config_with_equipment_appended(config: Config | None, equipment: EquipmentConfig) -> Config: + """Return a copy of ``config`` with ``equipment`` appended. + + The single place the Add-Equipment flow merges a new device into the + live config -- shared by the ``POST /config/equipment`` route and the + NiceGUI wizard's confirm step so both reject duplicate ids and re-run + the same cross-field validation instead of open-coding the merge + twice. ``config`` may be ``None`` on a fresh install that has no + ``config.yaml`` yet, in which case a default :class:`Config` is the + base. + + Raises :class:`exlab_wizard.errors.ConfigError` when an equipment + entry with the same id already exists. + """ + base = config or Config() + for entry in base.equipment: + if entry.id == equipment.id: + msg = f"equipment id {equipment.id!r} already exists in config" + raise ConfigError(msg) + merged = base.model_copy(update={"equipment": [*base.equipment, equipment]}) + # Re-run the full cross-field validation (unique ids, etc.) on the + # merged result so a bad merge fails loudly rather than persisting. + return Config.model_validate(merged.model_dump(mode="python")) diff --git a/src/exlab_wizard/constants/__init__.py b/src/exlab_wizard/constants/__init__.py index 659898a..c44da36 100644 --- a/src/exlab_wizard/constants/__init__.py +++ b/src/exlab_wizard/constants/__init__.py @@ -16,12 +16,10 @@ from exlab_wizard.constants.enums import ( AuditScopeKind, BandwidthDay, - CompletenessSignal, CreationLevel, DirectoryLevel, FieldType, FindingKind, - IngestState, LIMSProjectSource, LIMSProjectStatus, NextAction, @@ -32,6 +30,7 @@ ProblemClass, RunKind, RunScope, + RunSyncState, SessionKind, SetupNextAction, SetupState, @@ -54,7 +53,6 @@ COPIER_MANIFEST_NAME, CREATION_JSON_NAME, EQUIPMENT_JSON_NAME, - INGEST_JSON_NAME, LIMS_CACHE_DB_NAME, LOG_FILE_TEMPLATE, PLUGIN_MANIFEST_NAME, @@ -63,6 +61,7 @@ SECRETS_FILE, SERVER_STATE_FILE, SYNC_QUEUE_DB_NAME, + SYNC_STATE_FILENAME, TEST_RUNS_JSON_NAME, ) @@ -141,10 +140,10 @@ from exlab_wizard.constants.schema_versions import ( CREATION_JSON_VERSION, EQUIPMENT_JSON_VERSION, - INGEST_JSON_VERSION, OFFLINE_CATALOGUE_VERSION, README_FIELDS_JSON_VERSION, README_FRONT_MATTER_SCHEMA_VERSION, + SYNC_STATE_JSON_VERSION, TEST_RUNS_JSON_VERSION, ) @@ -173,8 +172,6 @@ "EQUIPMENT_ID_REGEX", "EQUIPMENT_JSON_NAME", "EQUIPMENT_JSON_VERSION", - "INGEST_JSON_NAME", - "INGEST_JSON_VERSION", # Keyring "KEYRING_SERVICE", "KEYRING_USERNAME_LIMS", @@ -223,6 +220,8 @@ "SESSION_GC_AFTER_SECONDS", "SIGTERM_DRAIN_TIMEOUT_SECONDS", "SYNC_QUEUE_DB_NAME", + "SYNC_STATE_FILENAME", + "SYNC_STATE_JSON_VERSION", "TEMPLATE_QUESTION_ID_PATTERN", "TEMPLATE_QUESTION_ID_REGEX", "TEST_RUNS_DIR_NAME", @@ -241,12 +240,10 @@ # Enum classes "AuditScopeKind", "BandwidthDay", - "CompletenessSignal", "CreationLevel", "DirectoryLevel", "FieldType", "FindingKind", - "IngestState", "LIMSProjectSource", "LIMSProjectStatus", "NextAction", @@ -257,6 +254,7 @@ "ProblemClass", "RunKind", "RunScope", + "RunSyncState", "SessionKind", "SetupNextAction", "SetupState", diff --git a/src/exlab_wizard/constants/enums.py b/src/exlab_wizard/constants/enums.py index 79565ee..532afa1 100644 --- a/src/exlab_wizard/constants/enums.py +++ b/src/exlab_wizard/constants/enums.py @@ -114,13 +114,23 @@ class LIMSProjectSource(StrEnum): OFFLINE_CATALOGUE = "offline_catalogue" -class IngestState(StrEnum): - """State machine for the NAS-ingest workflow. Backend Spec §13.3.""" +class RunSyncState(StrEnum): + """Derived run-level rollup of a run's per-file NAS sync progress. + + Operator-free per-file NAS sync design (2026-05-21). This rollup is + *never persisted*: it is computed on read from ``sync_state.json`` by + ``SyncStateWriter.rollup_state`` because the ``SYNCING``/``SYNCED`` + distinction can oscillate (a ``SYNCED`` run whose file is modified + again returns to ``SYNCING``). + + * ``SYNCING`` -- at least one tracked file is unverified, or no files are + tracked yet. + * ``SYNCED`` -- every tracked file has been verified on the NAS. + * ``CLEARED`` -- the run's staging copy has been cleaned up. + """ - STAGING = "staging" - COMPLETE = "complete" - SYNC_QUEUED = "sync_queued" - SYNC_VERIFIED = "sync_verified" + SYNCING = "syncing" + SYNCED = "synced" CLEARED = "cleared" @@ -166,16 +176,6 @@ class SyncMode(StrEnum): STAGE = "stage" -class CompletenessSignal(StrEnum): - """How a directory signals that its contents are finalized. - - Backend Spec §9 and §13.5. - """ - - SENTINEL_FILE = "sentinel_file" - MANIFEST = "manifest" - - class StagingCleanupMode(StrEnum): """How NAS staging directories are eventually purged. Backend Spec §13.7.""" diff --git a/src/exlab_wizard/constants/filenames.py b/src/exlab_wizard/constants/filenames.py index a798ad3..8e9ac22 100644 --- a/src/exlab_wizard/constants/filenames.py +++ b/src/exlab_wizard/constants/filenames.py @@ -20,8 +20,9 @@ # Cache filename for per-equipment static metadata. Backend Spec §11.4.1. EQUIPMENT_JSON_NAME: str = "equipment.json" -# Cache filename for the NAS ingest state machine. Backend Spec §13.4. -INGEST_JSON_NAME: str = "ingest.json" +# Cache filename for the per-run quiescence-driven per-file sync state. +# Operator-free per-file NAS sync design (2026-05-21). +SYNC_STATE_FILENAME: str = "sync_state.json" # Cache filename for the per-equipment test-run history. Backend Spec §11.4.2. TEST_RUNS_JSON_NAME: str = "test_runs.json" diff --git a/src/exlab_wizard/constants/schema_versions.py b/src/exlab_wizard/constants/schema_versions.py index d917bfa..543f7bc 100644 --- a/src/exlab_wizard/constants/schema_versions.py +++ b/src/exlab_wizard/constants/schema_versions.py @@ -14,9 +14,9 @@ # Version of the per-equipment ``readme_fields.json`` cache. Backend Spec §11.4. README_FIELDS_JSON_VERSION: str = "1.1" -# Version of the per-run ``ingest.json`` cache produced during NAS ingest. -# Backend Spec §13.4. -INGEST_JSON_VERSION: str = "1.1" +# Version of the per-run ``sync_state.json`` cache produced by the +# quiescence-driven per-file NAS sync poller (2026-05-21 design). +SYNC_STATE_JSON_VERSION: str = "1.0" # Version of the per-equipment ``equipment.json`` cache. Backend Spec §11.4.1. EQUIPMENT_JSON_VERSION: str = "1.0" diff --git a/src/exlab_wizard/controller/creation.py b/src/exlab_wizard/controller/creation.py index 6880c2e..9ae6caf 100644 --- a/src/exlab_wizard/controller/creation.py +++ b/src/exlab_wizard/controller/creation.py @@ -34,6 +34,7 @@ import asyncio import contextlib import shutil +import socket from collections.abc import AsyncIterator from dataclasses import dataclass, field from datetime import UTC, datetime @@ -52,7 +53,6 @@ ) from exlab_wizard.cache.creation_writer import CreationWriter from exlab_wizard.cache.equipment import EquipmentCacheWriter -from exlab_wizard.cache.ingest_writer import default_host from exlab_wizard.cache.log_writer import append_log_line, format_log_line from exlab_wizard.config.models import Config from exlab_wizard.constants import ( @@ -963,18 +963,14 @@ async def _write_cache( # Redesign §3.1: creation.json always carries the orchestrator # block. Redesign §3.3: the block carries the producing equipment's - # label + completeness-signal info so a receiving orchestrator - # can auto-discover the relayed equipment without a per-equipment - # config of its own. + # label so a receiving orchestrator can auto-discover the relayed + # equipment without a per-equipment config of its own. eq = next((e for e in self._config.equipment if e.id == req.equipment_id), None) orchestrator_block = OrchestratorBlock( enabled=True, - host=default_host(), + host=socket.gethostname(), label=self._config.orchestrator.label, equipment_label=eq.label if eq else None, - completeness_signal=eq.completeness_signal if eq else None, - sentinel_filename=eq.sentinel_filename if eq else None, - manifest_filename=eq.manifest_filename if eq else None, ) payload = CreationJson( diff --git a/src/exlab_wizard/orchestrator/__init__.py b/src/exlab_wizard/orchestrator/__init__.py index d8fb9f6..627b116 100644 --- a/src/exlab_wizard/orchestrator/__init__.py +++ b/src/exlab_wizard/orchestrator/__init__.py @@ -1,35 +1,28 @@ """Orchestrator-mode runtime. Backend Spec §12, §13. -This package implements the orchestrator-only features that activate when -``config.orchestrator.enabled`` is True: +This package implements the orchestrator-mode features that activate when a +``config.orchestrator.staging_root`` is configured or any ``nas``-mode +equipment exists: -* :class:`StagingWatcher` -- background polling task that walks - ``staging_root``, writes the initial ``ingest.json`` for each new run, - and drives the five-state lifecycle described in §13.3. -* :func:`cleanup_eligible` / :func:`clear_run` -- helpers for both the - manual operator flow and the scheduled background sweeper. +* :class:`QuiescenceSyncPoller` -- background polling task that discovers + every run pending NAS sync (orchestrator-staged *and* ``nas``-mode) and + enqueues a run once it has at least one quiescent file. It is the single + operator-free auto-sync trigger (operator-free per-file NAS sync design, + 2026-05-21), superseding the retired sentinel/manifest ``StagingWatcher``. * :func:`list_staged_runs` -- read-side query that backs the Staging UI panel and the ``GET /staging`` endpoint. - -The orchestrator never touches single-equipment workstations: every -public surface in the ``api/routers/staging.py`` router is gated behind -``config.orchestrator.enabled`` and returns 503 with -``code: "orchestrator_disabled"`` otherwise. """ from __future__ import annotations -from exlab_wizard.orchestrator.cleanup import cleanup_eligible, clear_run +from exlab_wizard.orchestrator.quiescence_poller import QuiescenceSyncPoller from exlab_wizard.orchestrator.staging_query import ( StagedRunSummary, list_staged_runs, ) -from exlab_wizard.orchestrator.staging_watcher import StagingWatcher __all__ = [ + "QuiescenceSyncPoller", "StagedRunSummary", - "StagingWatcher", - "cleanup_eligible", - "clear_run", "list_staged_runs", ] diff --git a/src/exlab_wizard/orchestrator/_scan.py b/src/exlab_wizard/orchestrator/_scan.py index f91a80e..b77c212 100644 --- a/src/exlab_wizard/orchestrator/_scan.py +++ b/src/exlab_wizard/orchestrator/_scan.py @@ -1,9 +1,10 @@ """Shared filesystem helpers for the orchestrator. Backend Spec §13.2. -Both :mod:`staging_query` and :mod:`staging_watcher` need to walk -``staging_root`` and discover run leaves; both also need to count files -and bytes under a run directory. Centralising these helpers keeps the -two modules in sync and avoids subtle drift in path conventions. +Both :mod:`staging_query` and :mod:`quiescence_poller` need to walk +``staging_root`` (and the ``nas``-mode equipment trees) to discover run +leaves; ``staging_query`` also counts files and bytes under a run +directory. Centralising these helpers keeps the modules in sync and +avoids subtle drift in path conventions. """ from __future__ import annotations @@ -21,6 +22,7 @@ __all__ = [ "count_files_and_bytes", "iter_subdirs", + "walk_equipment_run_leaves", "walk_run_leaves", ] @@ -50,35 +52,53 @@ def iter_subdirs(parent: Path) -> list[Path]: return out +def walk_equipment_run_leaves(equipment_dir: Path) -> list[Path]: + """Return every ``Run_*`` / ``TestRun_*`` directory under one equipment dir. + + ``equipment_dir`` is a single equipment's subtree + (``/``); runs live at ``//{Runs, + TestRuns}/`` per §13.2. GUI/Orchestrator Redesign §3.4 makes + experimental runs symmetric with test runs (both sit under a marker + folder). A misplaced ``Run_*`` directly under the project + (pre-redesign layout) is still surfaced so the validator can flag it. + + Distinct from :func:`walk_run_leaves`, which walks an equipment-first + *root* containing many equipment subtrees. The quiescence poller uses + this per-equipment form so a ``nas``-mode equipment's runs are + discovered without sweeping in a co-rooted ``stage``-mode equipment. + """ + leaves: list[Path] = [] + for project_dir in iter_subdirs(equipment_dir): + for child in iter_subdirs(project_dir): + if child.name == TEST_RUNS_DIR_NAME: + leaves.extend( + run_dir for run_dir in iter_subdirs(child) if is_test_run_dir(run_dir.name) + ) + elif child.name == RUNS_DIR_NAME: + leaves.extend( + run_dir for run_dir in iter_subdirs(child) if is_run_dir(run_dir.name) + ) + elif is_run_dir(child.name): + # Misplaced Run_* directly under the project — surface + # it so the validator's mode_prefix_mismatch rule can + # flag it as a hard finding. + leaves.append(child) + return leaves + + def walk_run_leaves(staging_root: Path) -> list[Path]: """Return every ``Run_*`` / ``TestRun_*`` directory under ``staging_root``. Per §13.2 the staging layout is - ``///{Runs,TestRuns}/``; - GUI/Orchestrator Redesign §3.4 makes experimental runs symmetric - with test runs (both sit under a marker folder). Misplaced - ``Run_*`` directly under the project (pre-redesign layout) is still - surfaced so the validator can flag it. + ``///{Runs,TestRuns}/``; this walks + an equipment-first *root* (many equipment subtrees). For a single + equipment's subtree use :func:`walk_equipment_run_leaves`. """ if not staging_root.exists(): return [] leaves: list[Path] = [] for equipment_dir in iter_subdirs(staging_root): - for project_dir in iter_subdirs(equipment_dir): - for child in iter_subdirs(project_dir): - if child.name == TEST_RUNS_DIR_NAME: - leaves.extend( - run_dir for run_dir in iter_subdirs(child) if is_test_run_dir(run_dir.name) - ) - elif child.name == RUNS_DIR_NAME: - leaves.extend( - run_dir for run_dir in iter_subdirs(child) if is_run_dir(run_dir.name) - ) - elif is_run_dir(child.name): - # Misplaced Run_* directly under the project — surface - # it so the validator's mode_prefix_mismatch rule can - # flag it as a hard finding. - leaves.append(child) + leaves.extend(walk_equipment_run_leaves(equipment_dir)) return leaves diff --git a/src/exlab_wizard/orchestrator/cleanup.py b/src/exlab_wizard/orchestrator/cleanup.py deleted file mode 100644 index 80d997b..0000000 --- a/src/exlab_wizard/orchestrator/cleanup.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Staging-side cleanup helpers. Backend Spec §13.7. - -Once a run is verified on the NAS, the orchestrator deletes the local -staging copy. Two policies are supported: - -* ``manual`` (default for v1) -- only an explicit operator action - advances ``sync_verified`` -> ``cleared``. The watcher never auto-clears. -* ``scheduled`` -- runs whose ``sync_verified_at`` was at least - ``retain_hours`` ago are auto-cleared by the periodic sweep. - -Deletion is logged with file count and bytes freed (§13.7). - -Both helpers are pure read-side utilities except :func:`clear_run`, -which performs the on-disk delete and writes the ``cleared`` history -entry. The watcher keeps the responsibility of *deciding* when to call -:func:`clear_run` -- this module only enforces the policy and the -filesystem effect. -""" - -from __future__ import annotations - -import shutil -from datetime import datetime, timedelta -from pathlib import Path - -from exlab_wizard.api.schemas import IngestJson -from exlab_wizard.cache.ingest_writer import IngestWriter, default_host -from exlab_wizard.config.models import Config -from exlab_wizard.constants import ( - IngestState, - StagingCleanupMode, -) -from exlab_wizard.logging import get_logger -from exlab_wizard.orchestrator.staging_query import list_staged_runs -from exlab_wizard.paths import ingest_json_path -from exlab_wizard.utils.time import parse_utc_iso_or_none, utc_now_or - -__all__ = [ - "cleanup_eligible", - "clear_all_verified", - "clear_run", - "freed_bytes_and_count", -] - -_log = get_logger(__name__) - - -def cleanup_eligible( - *, - ingest: IngestJson, - config: Config, - now_utc: datetime | None = None, -) -> bool: - """Return True if the run's local staging copy should be cleared now. - - Backend Spec §13.7: - - * ``manual`` -- always returns False. The operator must invoke - :func:`clear_run` directly (UI button or API). - * ``scheduled`` -- returns True iff ``current_state == sync_verified`` - AND ``sync_verified_at + retain_hours <= now_utc``. - - A run that is not ``sync_verified`` is never eligible -- attempting - to clear earlier states is a contract violation that the watcher - must avoid. - """ - if ingest.current_state != IngestState.SYNC_VERIFIED.value: - return False - mode = config.orchestrator.staging_cleanup.mode - if mode == StagingCleanupMode.MANUAL.value: - return False - if mode != StagingCleanupMode.SCHEDULED.value: - # Defensive: the Pydantic Literal already constrains the values, - # but if a future mode is added without a code path here we - # default to "not eligible" -- the safer behaviour. - return False - verified_at = _find_state_timestamp(ingest, IngestState.SYNC_VERIFIED) - if verified_at is None: - return False - now = utc_now_or(now_utc) - retain_hours = config.orchestrator.staging_cleanup.retain_hours - return verified_at + timedelta(hours=retain_hours) <= now - - -async def clear_run( - run_path: Path, - *, - config: Config, - ingest_writer: IngestWriter, - host: str | None = None, -) -> tuple[int, int]: - """Remove the staged run directory and append the ``cleared`` entry. - - Returns ``(file_count, bytes_freed)`` so the caller can log/notify - accurately. The ingest entry is written **before** the deletion so a - crash mid-clear leaves a coherent state record. - - The function is idempotent: calling it after the directory is gone - is a no-op that returns ``(0, 0)`` and does not append a duplicate - history entry. - """ - _ = config # kept on the signature for spec parity / future hooks - if not run_path.exists(): # noqa: ASYNC240 -- one-shot stat, sync filelock cycle below - return 0, 0 - file_count, bytes_freed = freed_bytes_and_count(run_path) - ingest_path = ingest_json_path(run_path) - host_label = host or default_host() - if ingest_path.exists(): - await ingest_writer.append_state_transition( - ingest_path, - IngestState.CLEARED, - host=host_label, - ) - # Now delete the staged directory in full -- the ingest.json entry - # we just wrote is part of the directory and is acceptable to discard - # because §13 only requires the ``cleared`` entry to flow to NAS via - # the prior ``sync_verified`` transition (the NAS copy already has it). - shutil.rmtree(run_path, ignore_errors=True) - _log.info( - "staging cleared: path=%s files=%d bytes_freed=%d host=%s", - run_path, - file_count, - bytes_freed, - host_label, - ) - return file_count, bytes_freed - - -async def clear_all_verified( - *, - config: Config, - ingest_writer: IngestWriter, - host: str | None = None, -) -> list[str]: - """Clear every staged run currently in ``sync_verified`` state. - - Backend Spec §4.6: the file-explorer footer's *Clear verified runs* - bulk action. Walks the staging tree via :func:`list_staged_runs`, - filters rows whose ``current_state`` is :data:`IngestState.SYNC_VERIFIED`, - and calls :func:`clear_run` on each. Returns the list of run paths - (as strings) that were cleared, in the order they were processed, - so the API layer can report a count to the operator. - - Errors from a single :func:`clear_run` are logged and skipped; the - bulk action proceeds with the remaining rows so one corrupted - staging entry can't block the whole sweep. - """ - cleared: list[str] = [] - for summary in list_staged_runs(config=config): - if summary.current_state != IngestState.SYNC_VERIFIED.value: - continue - run_path = Path(summary.path) - try: - files, _bytes = await clear_run( - run_path, - config=config, - ingest_writer=ingest_writer, - host=host, - ) - except Exception as exc: - # The watcher writes a coherent ingest.json before the rmtree - # so a mid-clear crash leaves a recoverable state. Bulk - # callers can retry; we don't let one failure abort the rest. - _log.warning( - "clear_all_verified: clear_run failed for %s: %s", - run_path, - exc, - ) - continue - if files > 0: - cleared.append(str(run_path)) - if cleared: - _log.info("clear_all_verified: cleared %d run(s)", len(cleared)) - return cleared - - -def freed_bytes_and_count(run_path: Path) -> tuple[int, int]: - """Sum file count and byte total under ``run_path``. - - Counts files only (directories are not counted as files); the - ``.exlab-wizard/`` cache subtree is included because :func:`clear_run` - deletes the whole run. - """ - files = 0 - total = 0 - if not run_path.exists(): - return 0, 0 - for entry in run_path.rglob("*"): - try: - if entry.is_file(): - files += 1 - total += entry.stat().st_size - except OSError: - continue - return files, total - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - - -def _find_state_timestamp(ingest: IngestJson, target: IngestState) -> datetime | None: - """Return the latest history-entry ``at`` for ``target`` or None.""" - for entry in reversed(ingest.history): - if entry.get("state") != target.value: - continue - return parse_utc_iso_or_none(entry.get("at")) - return None diff --git a/src/exlab_wizard/orchestrator/quiescence_poller.py b/src/exlab_wizard/orchestrator/quiescence_poller.py new file mode 100644 index 0000000..7eb4da3 --- /dev/null +++ b/src/exlab_wizard/orchestrator/quiescence_poller.py @@ -0,0 +1,358 @@ +"""Operator-free, per-file quiescence-driven NAS sync poller. + +Operator-free per-file NAS sync design (2026-05-21). The +:class:`QuiescenceSyncPoller` is the single auto-sync trigger for **every** +run pending NAS sync -- orchestrator-staged runs *and* runs acquired +directly on ``nas``-mode equipment. It supersedes the sentinel/manifest +``StagingWatcher`` and its five-state ``ingest.json`` machine. + +Each sweep (``poll_once``): + +1. **Discover runs** in both roots: + * stage-mode -- run-leaf directories under + ``config.orchestrator.staging_root``; + * nas-mode -- run-leaf directories under each ``nas``-mode equipment's + ``local_root`` tree (``///{Runs, + TestRuns}/``). +2. **Per-file quiescence.** The poller keeps an in-memory snapshot across + sweeps: for each file, its ``(st_size, st_mtime_ns)`` signature and the + wall-clock time the signature was *first observed*. A file is **quiet** + once that signature has been observed unchanged for at least + ``config.sync.quiescence_minutes``. Files matching any + ``config.sync.ignore_globs`` glob and the ``.exlab-wizard/`` cache dir + are skipped. Eligibility is measured from the poller's own observations + across sweeps -- *not* the absolute age of ``mtime`` -- because + transports (``rsync -t``, ``rclone``) preserve the source ``mtime``. +3. **Per-file eligibility.** A quiet file is *eligible* when its current + ``(st_size, st_mtime_ns)`` signature differs from the + ``synced_signature`` recorded for it in the run's ``sync_state.json`` + (a file with no record, or a record carrying a stale signature, is + eligible; a file matching its recorded signature has already synced at + its current state and is skipped). +4. **Enqueue.** Each discovered run with at least one eligible file is + enqueued via ``nas_sync.enqueue(run_path, files=[...])`` carrying the + run-relative paths of the eligible files. ``enqueue`` itself owns the + re-queue / no-op decision (a terminal job with a fresh subset is + re-armed; an active job is a no-op), so the poller no longer needs the + coarse "skip run with a job" guard. + +The poller is safe to cancel at any await point: it carries no on-disk +state of its own (the snapshot is purely in-memory and is rebuilt by +re-observing the filesystem on the next sweep). +""" + +from __future__ import annotations + +import asyncio +import contextlib +import fnmatch +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol + +from exlab_wizard.config.models import Config +from exlab_wizard.constants import CACHE_DIR_NAME, SyncMode +from exlab_wizard.logging import get_logger +from exlab_wizard.orchestrator._scan import walk_equipment_run_leaves, walk_run_leaves + +if TYPE_CHECKING: + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + +__all__ = ["FileSnapshot", "NASSyncLike", "QuiescenceSyncPoller"] + +_log = get_logger(__name__) + + +# --------------------------------------------------------------------------- +# Protocols (kept loose so production NASSyncClient and test stubs both fit) +# --------------------------------------------------------------------------- + + +class NASSyncLike(Protocol): + """The NAS-sync surface the poller and the Phase 3 staging code use. + + ``enqueue`` is what the poller itself calls; the ``staging`` router and + ``ui/mount`` run-status code additionally consult ``status`` / + ``get_by_run_path`` / ``list_all``. They are declared here so the + protocol documents the full surface that ``deps.nas_sync`` must + satisfy. Tests pass in-memory stubs that record the calls. + """ + + async def enqueue(self, run_path: Path, files: list[str] | None = ...) -> Any: ... + async def status(self, run_path: Path) -> str: ... + async def get_by_run_path(self, run_path: Path) -> Any: ... + async def list_all(self) -> Any: ... + + +@dataclass(slots=True) +class FileSnapshot: + """One file's observation record carried forward across sweeps. + + * ``signature`` -- the ``(st_size, st_mtime_ns)`` last observed. + * ``first_seen_monotonic`` -- the ``time.monotonic()`` value at which + that exact signature was *first* observed. Reset whenever the + signature changes; the settle window is measured from it. + """ + + signature: tuple[int, int] + first_seen_monotonic: float + + +class QuiescenceSyncPoller: + """Polls every run pending NAS sync and enqueues runs with eligible files. + + Constructor dependencies are a :class:`Config`, a + :class:`NASSyncClient`-shaped sync client, and a :class:`SyncStateWriter` + used (read-only) to skip files already synced at their current + ``(st_size, st_mtime_ns)`` signature. + + The start/stop/loop lifecycle mirrors the retired ``StagingWatcher``; + ``poll_once`` is exposed so tests can drive the poller synchronously. + """ + + def __init__( + self, + *, + config: Config, + nas_sync: NASSyncLike, + sync_state_writer: SyncStateWriter, + ) -> None: + self._config = config + self._nas_sync = nas_sync + self._sync_state_writer = sync_state_writer + self._task: asyncio.Task[None] | None = None + self._stopping = False + # Per-file observation snapshot, keyed by absolute path, carried + # across sweeps so the settle window can be measured. + self._snapshots: dict[Path, FileSnapshot] = {} + + # ------------------------------------------------------------------ lifecycle + + async def start(self) -> None: + """Start the background polling task. Idempotent. + + Returns immediately; the task runs until :meth:`stop` is called or + the surrounding event loop tears down. + """ + if self._task is not None and not self._task.done(): + return + self._stopping = False + self._task = asyncio.create_task(self._loop(), name="exlab-quiescence-poller") + _log.info( + "quiescence poller started: poll_interval_s=%d quiescence_minutes=%d", + self._config.sync.poll_interval_seconds, + self._config.sync.quiescence_minutes, + ) + + async def stop(self) -> None: + """Cancel the background task and wait for it to exit. Idempotent.""" + self._stopping = True + if self._task is None: + return + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._task + self._task = None + _log.info("quiescence poller stopped") + + # ------------------------------------------------------------------ poll loop + + async def _loop(self) -> None: + interval = float(self._config.sync.poll_interval_seconds) + try: + while not self._stopping: + try: + await self.poll_once() + except asyncio.CancelledError: + raise + except Exception: # pragma: no cover -- defensive + _log.exception("quiescence poller sweep failed") + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for( + asyncio.shield(asyncio.sleep(interval)), + timeout=interval + 1.0, + ) + except asyncio.CancelledError: + raise + + async def poll_once(self, *, now_monotonic: float | None = None) -> list[Path]: + """Run one discovery + quiescence + enqueue sweep. + + ``now_monotonic`` is injectable so tests can drive the settle + window with a controllable clock; production callers leave it + ``None`` and the poller reads ``time.monotonic()``. + + Returns the list of run paths enqueued on this sweep (in discovery + order) so tests can assert exactly which runs fired. + """ + now = time.monotonic() if now_monotonic is None else now_monotonic + runs = self._discover_runs() + settle_seconds = self._config.sync.quiescence_minutes * 60 + # Build the next-sweep snapshot fresh; files that vanished simply + # fall out of the carried-forward dict. + next_snapshots: dict[Path, FileSnapshot] = {} + enqueued: list[Path] = [] + for run_path in runs: + quiet_files: list[Path] = [] + for file_path in self._iter_run_files(run_path): + signature = self._signature(file_path) + if signature is None: + continue + prior = self._snapshots.get(file_path) + if prior is not None and prior.signature == signature: + snap = FileSnapshot(signature, prior.first_seen_monotonic) + else: + snap = FileSnapshot(signature, now) + next_snapshots[file_path] = snap + if now - snap.first_seen_monotonic >= settle_seconds: + quiet_files.append(file_path) + if not quiet_files: + continue + # Per-file eligibility: a quiet file is enqueued only when its + # current signature differs from the ``synced_signature`` + # recorded in ``sync_state.json`` (a file with no record, or a + # record with a stale signature, is eligible). ``enqueue`` + # itself owns the re-queue / no-op decision against any + # existing job, so the poller no longer pre-filters on job + # state. + eligible = await self._eligible_files(run_path, quiet_files) + if eligible: + await self._nas_sync.enqueue(run_path, eligible) + enqueued.append(run_path) + self._snapshots = next_snapshots + return enqueued + + # ------------------------------------------------------------------ discovery + + def _discover_runs(self) -> list[Path]: + """Return every run-leaf directory pending NAS sync. + + Stage-mode runs sit under ``orchestrator.staging_root`` (an + equipment-first root). Nas-mode runs sit under each ``nas``-mode + equipment's *own* subtree ``/`` -- NOT + the whole ``local_root`` tree, which is shared across equipment + (a co-rooted ``stage``-mode equipment's runs must not be swept in + here, they reach the NAS via the orchestrator's staging area). A + run discovered through both roots is de-duplicated. + """ + seen: set[Path] = set() + runs: list[Path] = [] + + def _add(leaf: Path) -> None: + resolved = _safe_resolve(leaf) + if resolved in seen: + return + seen.add(resolved) + runs.append(leaf) + + staging_root = self._config.orchestrator.staging_root + if staging_root: + for leaf in walk_run_leaves(Path(staging_root)): + _add(leaf) + for equipment in self._config.equipment: + if equipment.sync_mode != SyncMode.NAS: + continue + if not equipment.local_root: + continue + # Runs live at ``///...``; + # walk only this equipment's own subtree so a co-rooted + # ``stage``-mode equipment is never discovered here. + equipment_dir = Path(equipment.local_root) / equipment.id + for leaf in walk_equipment_run_leaves(equipment_dir): + _add(leaf) + return runs + + # ------------------------------------------------------------------ quiescence + + def _iter_run_files(self, run_path: Path) -> list[Path]: + """Return every non-ignored file under ``run_path``. + + Skips the ``.exlab-wizard/`` cache dir and any file whose name + matches a ``config.sync.ignore_globs`` glob. + """ + ignore_globs = self._config.sync.ignore_globs + out: list[Path] = [] + stack: list[Path] = [run_path] + while stack: + current = stack.pop() + try: + entries = list(os.scandir(current)) + except (FileNotFoundError, NotADirectoryError, PermissionError): + continue + for entry in entries: + if entry.name == CACHE_DIR_NAME and current == run_path: + continue + try: + if entry.is_dir(follow_symlinks=False): + stack.append(Path(entry.path)) + continue + if not entry.is_file(follow_symlinks=False): + continue + except OSError: + continue + if _matches_any_glob(entry.name, ignore_globs): + continue + out.append(Path(entry.path)) + return out + + @staticmethod + def _signature(file_path: Path) -> tuple[int, int] | None: + """Return ``(st_size, st_mtime_ns)`` for ``file_path`` or None.""" + try: + stat = file_path.stat() + except OSError: + return None + return (stat.st_size, stat.st_mtime_ns) + + async def _eligible_files( + self, + run_path: Path, + quiet_files: list[Path], + ) -> list[str]: + """Return the run-relative paths of quiet files needing a (re-)sync. + + A quiet file is eligible when its current ``(st_size, + st_mtime_ns)`` signature differs from the ``synced_signature`` + recorded for it in the run's ``sync_state.json`` -- i.e. it has + never synced, or it was modified after a prior sync. A file whose + signature matches its recorded ``synced_signature`` has already + synced at its current state and is skipped. + + Returns paths sorted for determinism. On a ``sync_state.json`` read + failure the poller treats every quiet file as eligible (fail-open: + re-syncing an already-synced file is wasteful but safe). + """ + try: + state = await self._sync_state_writer.read(run_path) + except Exception as exc: # pragma: no cover -- defensive + _log.warning("sync_state.json read failed for %s: %s", run_path, exc) + synced: dict[str, tuple[int, int] | None] = {} + else: + synced = {rel: rec.synced_signature for rel, rec in state.files.items()} + + eligible: list[str] = [] + for file_path in quiet_files: + signature = self._signature(file_path) + if signature is None: + continue + rel = file_path.relative_to(run_path).as_posix() + recorded = synced.get(rel) + if recorded is not None and tuple(recorded) == signature: + continue + eligible.append(rel) + return sorted(eligible) + + +def _matches_any_glob(name: str, globs: list[str]) -> bool: + """Return True if ``name`` matches any glob in ``globs``.""" + return any(fnmatch.fnmatch(name, pattern) for pattern in globs) + + +def _safe_resolve(path: Path) -> Path: + """Resolve ``path`` for de-dup, falling back to the path itself.""" + try: + return path.resolve() + except OSError: + return path diff --git a/src/exlab_wizard/orchestrator/staging_clear.py b/src/exlab_wizard/orchestrator/staging_clear.py new file mode 100644 index 0000000..3dee8ba --- /dev/null +++ b/src/exlab_wizard/orchestrator/staging_clear.py @@ -0,0 +1,80 @@ +"""Operator-facing staging-clear helper. Backend Spec §13.7. + +The operator-free per-file NAS sync redesign (2026-05-21) removed the +``orchestrator.cleanup`` module and the ``ingest.json`` state machine. Two +call sites -- the ``/staging`` router and the NiceGUI mount -- delete a +staged run's local copy on operator request ("Clear" / "Clear verified +runs"). + +Phase 5: an operator clear must have the **same** keep-local-aware +semantics as the automatic cleanup reaper +(:meth:`exlab_wizard.sync.nas_client.NASSyncClient._delete_local`). Both +delegate to the shared :func:`exlab_wizard.sync.run_delete.delete_run_files` +helper, so a ``keep_local`` file survives an operator clear and the +``.exlab-wizard/`` metadata subtree (incl. ``sync_state.json``) is retained +-- a cleared run still renders its files as "On NAS" tombstones. After the +delete, ``cleared_at`` is stamped so the run's rollup flips to ``CLEARED``. +""" + +from __future__ import annotations + +from pathlib import Path + +from exlab_wizard.cache.sync_state_writer import SyncStateWriter +from exlab_wizard.logging import get_logger +from exlab_wizard.orchestrator._scan import count_files_and_bytes +from exlab_wizard.sync.run_delete import delete_run_files + +__all__ = ["clear_run_dir"] + +_log = get_logger(__name__) + + +def clear_run_dir(run_path: Path) -> tuple[int, int]: + """Delete a staged run's data files; return ``(file_count, bytes_freed)``. + + Idempotent: a missing directory returns ``(0, 0)``. + + Keep-local-aware (Phase 5): files flagged ``keep_local`` in the run's + ``sync_state.json`` survive, the ``.exlab-wizard/`` metadata subtree is + retained, and directory symlinks are never descended into or removed -- + identical semantics to the automatic cleanup reaper. After deletion, + ``cleared_at`` is stamped in ``sync_state.json`` so the run rolls up to + ``CLEARED`` (and still lists "On NAS" tombstones). + + The reported ``(file_count, bytes_freed)`` counts the files actually + removed -- i.e. excludes the cache subtree and any retained ``keep_local`` + files. + """ + if not run_path.exists(): + return 0, 0 + + writer = SyncStateWriter() + state = writer.read_sync(run_path) + keep_local = {rel for rel, rec in state.files.items() if rec.keep_local} + + # Count only the files that will actually be removed: total run files + # (excluding the cache subtree) minus the retained keep-local files. + file_count, bytes_freed = count_files_and_bytes(run_path, exclude_cache=True) + for rel in keep_local: + kept = run_path / rel + try: + stat = kept.stat() + except OSError: + continue + file_count -= 1 + bytes_freed -= stat.st_size + + delete_run_files(run_path, keep_local=keep_local, retain_cache=True) + # Stamp ``cleared_at`` so the run rolls up to CLEARED. ``retain_cache`` + # is forced True above, so the cache subtree (and the record) survive. + writer.mark_cleared_sync(run_path) + + _log.info( + "staging cleared: path=%s files=%d bytes_freed=%d keep_local=%d", + run_path, + max(file_count, 0), + max(bytes_freed, 0), + len(keep_local), + ) + return max(file_count, 0), max(bytes_freed, 0) diff --git a/src/exlab_wizard/orchestrator/staging_query.py b/src/exlab_wizard/orchestrator/staging_query.py index b2cb386..cdcaaa9 100644 --- a/src/exlab_wizard/orchestrator/staging_query.py +++ b/src/exlab_wizard/orchestrator/staging_query.py @@ -1,20 +1,24 @@ -"""Read-only enumeration of staged runs. Backend Spec §13.8. +"""Read-only enumeration of runs pending NAS sync. Backend Spec §13.8. The orchestrator exposes one read-side query that walks the configured -``staging_root``, opens each run's ``ingest.json``, and returns a small +``staging_root``, discovers every run-leaf directory, and returns a small DTO per run. This data backs both the bottom-dock UI panel and the ``GET /staging`` endpoint. Per §13.2 the staging tree mirrors the final NAS layout -(``///Run_`` or -``///TestRuns/TestRun_``). The walker -descends to the run-leaf directory, looks for ``.exlab-wizard/ingest.json``, -and skips any directory that lacks one (the staging push has not yet -written the initial state record). - -The query returns rows sorted by "most recent activity first" -- defined -as the timestamp of the most recent ``history`` entry. Runs without a -parsable history fall back to the directory's mtime. +(``///Runs/Run_`` or +``///TestRuns/TestRun_``). Equipment id, +project name, and run kind are derived from the run path itself. + +The operator-free per-file NAS sync redesign (2026-05-21) removed +``ingest.json``; a run's lifecycle ``current_state`` is the derived +``sync_state.json`` rollup -- ``syncing`` / ``synced`` / ``cleared`` +(:class:`~exlab_wizard.constants.RunSyncState`), computed by +:func:`SyncStateWriter.rollup_state` from the per-run +``/.exlab-wizard/sync_state.json`` record. A run with no +``sync_state.json`` yet (no sync activity) rolls up to ``syncing``. + +The query returns rows sorted by directory mtime, most recent first. """ from __future__ import annotations @@ -23,20 +27,13 @@ from datetime import UTC, datetime from pathlib import Path -import msgspec - -from exlab_wizard.api.schemas import IngestJson +from exlab_wizard.cache.sync_state_writer import SyncStateWriter from exlab_wizard.config.models import Config -from exlab_wizard.io import read_msgspec_json +from exlab_wizard.constants import RUNS_DIR_NAME, TEST_RUNS_DIR_NAME, RunKind, RunSyncState from exlab_wizard.logging import get_logger from exlab_wizard.orchestrator._scan import count_files_and_bytes, walk_run_leaves -from exlab_wizard.paths import ingest_json_path -from exlab_wizard.utils.time import ( - dt_to_iso, - parse_utc_iso_or_none, - utc_now_iso, - utc_now_or, -) +from exlab_wizard.paths import is_test_run_dir +from exlab_wizard.utils.time import dt_to_iso, parse_utc_iso_or_none, utc_now_iso, utc_now_or __all__ = ["StagedRunSummary", "list_staged_runs"] @@ -50,15 +47,16 @@ class StagedRunSummary: Backend Spec §13.8: * ``path`` -- absolute filesystem path of the run leaf directory. - * ``current_state`` -- the latest ``ingest.json`` ``current_state``. + * ``current_state`` -- the run's derived ``sync_state.json`` rollup + (:class:`~exlab_wizard.constants.RunSyncState` value: + ``"syncing"`` / ``"synced"`` / ``"cleared"``). * ``equipment_id`` -- the equipment segment of the run path. * ``project_name`` -- the LIMS project short id (parent dir). * ``run_kind`` -- ``"experimental"`` or ``"test"``. * ``file_count`` / ``byte_total`` -- size of the staged data. * ``elapsed_seconds_since_last_activity`` -- seconds between - ``now_utc`` and the most recent history entry's ``at`` field - (falls back to the directory mtime when no history exists). - * ``last_activity_at`` -- ISO-8601 string of the same timestamp. + ``now_utc`` and the run directory's mtime. + * ``last_activity_at`` -- ISO-8601 string of the directory mtime. """ path: str @@ -77,13 +75,25 @@ def list_staged_runs( config: Config, staging_root: Path | None = None, now_utc: datetime | None = None, + sync_state_writer: SyncStateWriter | None = None, ) -> list[StagedRunSummary]: - """Enumerate every staged run with its current lifecycle state. + """Enumerate every staged run with its derived lifecycle state. ``staging_root`` defaults to ``config.orchestrator.staging_root``. Returns an empty list when ``staging_root`` is unset / missing. - Sort order: most recent activity first. + ``current_state`` is the derived ``sync_state.json`` rollup + (:class:`~exlab_wizard.constants.RunSyncState`): ``"syncing"`` / + ``"synced"`` / ``"cleared"``. ``sync_state_writer`` is used to read + each run's per-file record; a default :class:`SyncStateWriter` is + constructed when the caller omits it. A run without a + ``sync_state.json`` (no sync activity yet) rolls up to ``"syncing"``. + + This function is synchronous: it reads ``sync_state.json`` via the + writer's blocking :meth:`SyncStateWriter.read_sync` so sync NiceGUI + page handlers can call it without an event loop. + + Sort order: most recent directory mtime first. """ if staging_root is not None: root = staging_root @@ -94,11 +104,8 @@ def list_staged_runs( if not root.exists(): return [] now = utc_now_or(now_utc) - rows = [ - summary - for run_path in walk_run_leaves(root) - if (summary := _summarize_run(run_path, now)) is not None - ] + writer = sync_state_writer if sync_state_writer is not None else SyncStateWriter() + rows = [_summarize_run(run_path, root, now, writer) for run_path in walk_run_leaves(root)] # Sort by last activity desc; ties broken by path for determinism. rows.sort(key=lambda s: (-_iso_to_epoch(s.last_activity_at), s.path)) return rows @@ -109,33 +116,29 @@ def list_staged_runs( # --------------------------------------------------------------------------- -def _summarize_run(run_path: Path, now: datetime) -> StagedRunSummary | None: - """Build a :class:`StagedRunSummary` for ``run_path``, or None to skip. +def _summarize_run( + run_path: Path, + staging_root: Path, + now: datetime, + sync_state_writer: SyncStateWriter, +) -> StagedRunSummary: + """Build a :class:`StagedRunSummary` for ``run_path``. - A run is included only if its ``.exlab-wizard/ingest.json`` exists and - decodes successfully. Other shapes (a run that's mid-push and hasn't - written its initial ingest yet) are silently omitted -- the watcher - will catch up on the next pass. + Equipment id / project name / run kind are derived from the run path + relative to ``staging_root`` (``//{Runs,TestRuns}/``). + ``current_state`` is the derived rollup of the run's ``sync_state.json``. """ - ingest_path = ingest_json_path(run_path) - if not ingest_path.exists(): - return None - try: - ingest = read_msgspec_json(ingest_path, IngestJson) - except (msgspec.DecodeError, msgspec.ValidationError) as exc: - _log.warning("ingest.json at %s could not be decoded: %s", ingest_path, exc) - return None - + equipment_id, project_name, run_kind = _path_identity(run_path, staging_root) file_count, byte_total = count_files_and_bytes(run_path, exclude_cache=True) - last_activity_at = _last_activity_at(ingest, run_path) + last_activity_at = _dir_mtime_iso(run_path) elapsed = max(int((now - _parse_iso(last_activity_at, fallback=now)).total_seconds()), 0) - project_name = ingest.project_name or run_path.parent.name + current_state = _rollup_value(run_path, sync_state_writer) return StagedRunSummary( path=str(run_path), - current_state=ingest.current_state, - equipment_id=ingest.equipment_id, + current_state=current_state, + equipment_id=equipment_id, project_name=project_name, - run_kind=ingest.run_kind, + run_kind=run_kind, file_count=file_count, byte_total=byte_total, elapsed_seconds_since_last_activity=elapsed, @@ -143,19 +146,48 @@ def _summarize_run(run_path: Path, now: datetime) -> StagedRunSummary | None: ) -def _last_activity_at(ingest: IngestJson, run_path: Path) -> str: - """Return the ISO timestamp of the most recent activity. +def _rollup_value(run_path: Path, sync_state_writer: SyncStateWriter) -> str: + """Return the derived ``sync_state.json`` rollup for ``run_path``. - Preference order: + Reads the run's ``sync_state.json`` and derives the + :class:`~exlab_wizard.constants.RunSyncState` rollup. Any read error + (missing/locked/corrupt record) degrades to ``"syncing"`` -- the + safe default that keeps an unproven run out of the clearable set. + """ + try: + state = sync_state_writer.read_sync(run_path) + except Exception as exc: # pragma: no cover -- defensive + _log.warning("sync_state.json read failed for %s: %s", run_path, exc) + return RunSyncState.SYNCING.value + return sync_state_writer.rollup_state(state).value - 1. Most recent ``history`` entry's ``at`` field. - 2. The directory's mtime, formatted as UTC ISO. + +def _path_identity(run_path: Path, staging_root: Path) -> tuple[str, str, str]: + """Return ``(equipment_id, project_name, run_kind)`` from the run path. + + Falls back to empty strings / ``experimental`` when the path does not + sit cleanly under ``staging_root`` in the expected layout. """ - if ingest.history: - last = ingest.history[-1] - at = last.get("at") - if isinstance(at, str) and at: - return at + try: + relative = run_path.resolve().relative_to(staging_root.resolve()) + except (ValueError, OSError): + relative = Path(run_path.name) + parts = relative.parts + equipment_id = parts[0] if parts else "" + project_name = parts[1] if len(parts) >= 2 else run_path.parent.name + # The run kind is determined by the marker folder (Runs / TestRuns) or + # the leaf name prefix as a fallback. + if TEST_RUNS_DIR_NAME in parts or is_test_run_dir(run_path.name): + run_kind = RunKind.TEST.value + elif RUNS_DIR_NAME in parts: + run_kind = RunKind.EXPERIMENTAL.value + else: + run_kind = RunKind.EXPERIMENTAL.value + return equipment_id, project_name, run_kind + + +def _dir_mtime_iso(run_path: Path) -> str: + """Return the run directory mtime as a UTC ISO-8601 string.""" try: mtime = run_path.stat().st_mtime except OSError: diff --git a/src/exlab_wizard/orchestrator/staging_watcher.py b/src/exlab_wizard/orchestrator/staging_watcher.py deleted file mode 100644 index c08e342..0000000 --- a/src/exlab_wizard/orchestrator/staging_watcher.py +++ /dev/null @@ -1,546 +0,0 @@ -"""Background staging watcher. Backend Spec §12, §13. - -The :class:`StagingWatcher` is a polling task that drives the five-state -lifecycle (§13.3) for every directory under ``staging_root``: - - staging -> complete -> sync_queued -> sync_verified -> cleared - -For each newly-discovered run: - -1. Read the equipment-side ``creation.json`` already pushed into - ``/.exlab-wizard/creation.json`` and produce the initial - ``ingest.json`` with ``current_state == staging``. -2. Watch the run for the configured completeness signal (sentinel file - or manifest comparison; §13.5). Promote ``staging`` -> ``complete``. -3. Enqueue with the supplied :class:`NASSyncClient`. On a successful - enqueue, promote ``complete`` -> ``sync_queued``. -4. Poll the NAS sync status. On ``verified`` (or its post-cleanup - states), promote ``sync_queued`` -> ``sync_verified``. -5. If the cleanup policy is ``scheduled`` and ``retain_hours`` has - elapsed since ``sync_verified``, invoke :func:`clear_run` and - promote ``sync_verified`` -> ``cleared``. ``manual`` mode never - auto-clears -- the operator must invoke the action explicitly. - -The watcher is designed to be safe to cancel at any await point: state -transitions land on disk in a single locked write per :class:`IngestWriter`, -so a partial run leaves a coherent file. Re-running the loop picks up -from the on-disk state. - -The watcher is **not** the place where the run completeness is decided -in the production sense -- the equipment machine pushes a sentinel file -or manifest, and the watcher merely observes its presence. This keeps -transport and acquisition policy out of the app per §13.6. -""" - -from __future__ import annotations - -import asyncio -import contextlib -from collections.abc import Awaitable, Callable -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Protocol - -import msgspec - -from exlab_wizard.api.schemas import CreationJson, IngestJson -from exlab_wizard.cache.ingest_writer import IngestWriter, default_host -from exlab_wizard.config.models import Config, EquipmentConfig -from exlab_wizard.constants import ( - INGEST_JSON_VERSION, - CompletenessSignal, - IngestState, - OrchestratorTransportType, - RunKind, -) -from exlab_wizard.io import read_msgspec_json_raw -from exlab_wizard.logging import get_logger -from exlab_wizard.orchestrator._scan import ( - count_files_and_bytes, - walk_run_leaves, -) -from exlab_wizard.orchestrator.cleanup import cleanup_eligible, clear_run -from exlab_wizard.paths import ( - creation_json_path, - ingest_json_path, - is_run_dir, - is_test_run_dir, -) -from exlab_wizard.sync.queue import SyncJobState -from exlab_wizard.utils.time import utc_now_iso - -__all__ = ["NASSyncLike", "StagingWatcher"] - -_log = get_logger(__name__) - - -# --------------------------------------------------------------------------- -# Protocols (kept loose so production NASSyncClient and test stubs both fit) -# --------------------------------------------------------------------------- - - -class NASSyncLike(Protocol): - """The subset of :class:`NASSyncClient` the watcher uses. - - Only ``enqueue(run_path)`` and ``status(run_path)`` are needed. - Tests pass an in-memory stub that records the calls. - """ - - async def enqueue(self, run_path: Path) -> Any: ... - async def status(self, run_path: Path) -> str: ... - - -class CreationCacheLike(Protocol): - """The subset of :class:`CreationWriter` the watcher uses.""" - - async def read_creation_snapshot(self, path: Path) -> CreationJson: ... - - -# Verified statuses reported by NASSyncClient.status() (see Backend Spec §7.1.2). -# Anything in this set means the NAS copy is durably present. Derived from the -# queue-internal SyncJobState rather than a separate string set so the status -# values stay in sync with the queue's state machine (Backend Spec §7.1.2). -_VERIFIED_STATUSES: frozenset[str] = frozenset( - { - SyncJobState.VERIFIED.value, - SyncJobState.CLEANUP_ELIGIBLE.value, - SyncJobState.CLEANED.value, - }, -) - - -@dataclass -class _RunLocator: - """Computed values for the staged run we are evaluating.""" - - run_path: Path - cache_dir: Path - creation_path: Path - ingest_path: Path - equipment_id: str - project_name: str - run_kind: RunKind - - -class StagingWatcher: - """Polls ``staging_root`` and drives the §13.3 lifecycle. - - Constructor arguments mirror the spec: a :class:`Config`, the - orchestrator-side :class:`IngestWriter`, a :class:`NASSyncClient`-shaped - sync client, the :class:`CreationWriter` used to read pushed creation - snapshots, and an optional ``on_state_change`` callable invoked after - every successful transition (used by the UI to refresh the panel). - - Polling cadence defaults to 10s (§13.5 -- "polls until all are - present") and is overridable for tests via ``poll_interval_s``. - """ - - def __init__( - self, - *, - config: Config, - ingest_writer: IngestWriter, - nas_sync: NASSyncLike, - cache_creation: CreationCacheLike, - on_state_change: Callable[[Path, IngestState], Awaitable[None] | None] | None = None, - poll_interval_s: float = 10.0, - ) -> None: - self._config = config - self._ingest = ingest_writer - self._nas_sync = nas_sync - self._cache_creation = cache_creation - self._on_state_change = on_state_change - self._poll_interval_s = poll_interval_s - self._task: asyncio.Task[None] | None = None - self._stopping = False - self._equipment_by_id: dict[str, EquipmentConfig] = {e.id: e for e in config.equipment} - - # ------------------------------------------------------------------ lifecycle - - async def start(self) -> None: - """Start the background polling task. Idempotent. - - Returns immediately; the task runs until :meth:`stop` is called - or the surrounding event loop tears down. - """ - if self._task is not None and not self._task.done(): - return - # Redesign §3.1: orchestrator pipeline is always active. The watcher - # starts unconditionally; it stays a no-op when the configured - # staging_root does not exist (handled per-poll in poll_once). - self._stopping = False - self._task = asyncio.create_task(self._loop(), name="exlab-staging-watcher") - _log.info( - "staging watcher started: staging_root=%s poll_interval_s=%.1f", - self._config.orchestrator.staging_root, - self._poll_interval_s, - ) - - async def stop(self) -> None: - """Cancel the background task and wait for it to exit. Idempotent.""" - self._stopping = True - if self._task is None: - return - self._task.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await self._task - self._task = None - _log.info("staging watcher stopped") - - # ------------------------------------------------------------------ poll loop - - async def _loop(self) -> None: - try: - while not self._stopping: - try: - await self.poll_once() - except asyncio.CancelledError: - raise - except Exception: # pragma: no cover -- defensive - _log.exception("staging watcher poll failed") - with contextlib.suppress(asyncio.TimeoutError): - await asyncio.wait_for( - asyncio.shield(asyncio.sleep(self._poll_interval_s)), - timeout=self._poll_interval_s + 1.0, - ) - except asyncio.CancelledError: - raise - - async def poll_once(self) -> list[IngestState]: - """Walk staging_root once and run :meth:`evaluate_run` on each leaf. - - Exposed publicly so tests can drive the watcher synchronously - without spinning up the asyncio task. Returns the list of - post-evaluation states for every run found (in walk order). - """ - staging_root = Path(self._config.orchestrator.staging_root) - if not staging_root.exists(): # noqa: ASYNC240 -- one-shot stat - return [] - states: list[IngestState] = [] - for run_path in self._walk_run_leaves(staging_root): - try: - state = await self.evaluate_run(run_path) - except Exception as exc: # pragma: no cover -- defensive - _log.warning("evaluate_run failed for %s: %s", run_path, exc) - continue - states.append(state) - return states - - # ------------------------------------------------------------------ evaluate_run - - async def evaluate_run(self, run_path: Path) -> IngestState: - """Evaluate one run and advance its state if conditions are met. - - Returns the post-evaluation state. Idempotent within a single - cycle -- calling repeatedly while no condition has changed is a - cheap no-op (only reads the on-disk ingest entry). - - State decisions, in order: - - 1. No ``ingest.json`` yet -- bootstrap one in ``staging`` from - the equipment's pushed ``creation.json``. - 2. ``staging`` -- if the equipment-defined completeness signal is - present, advance to ``complete`` (recording file/byte counts). - 3. ``complete`` -- enqueue with NASSyncClient; on success - advance to ``sync_queued``. - 4. ``sync_queued`` -- check NAS sync status; on verified-or-better - advance to ``sync_verified``. - 5. ``sync_verified`` -- if cleanup policy says we may clear, - invoke :func:`clear_run` and advance to ``cleared``. - 6. ``cleared`` -- terminal; nothing more to do. - """ - loc = self._locator_for(run_path) - if loc is None: - return IngestState.STAGING # unrecognised shape; reported as no-op - - # Bootstrap an ingest.json if missing. - if not loc.ingest_path.exists(): - await self._bootstrap_initial_ingest(loc) - return IngestState.STAGING - - ingest = await self._ingest.read_ingest(loc.ingest_path) - current = IngestState(ingest.current_state) - match current: - case IngestState.STAGING: - if await self._completeness_signal_present(loc): - files, bytes_received = self._count_files_and_bytes(run_path) - await self._advance( - loc, - next_state=IngestState.COMPLETE, - files_received=files, - bytes_received=bytes_received, - ) - return IngestState.COMPLETE - return IngestState.STAGING - - case IngestState.COMPLETE: - await self._nas_sync.enqueue(run_path) - await self._advance(loc, next_state=IngestState.SYNC_QUEUED) - return IngestState.SYNC_QUEUED - - case IngestState.SYNC_QUEUED: - status = await self._nas_sync.status(run_path) - if status in _VERIFIED_STATUSES: - await self._advance( - loc, - next_state=IngestState.SYNC_VERIFIED, - nas_path=ingest.run_path or "", - ) - return IngestState.SYNC_VERIFIED - return IngestState.SYNC_QUEUED - - case IngestState.SYNC_VERIFIED: - if cleanup_eligible(ingest=ingest, config=self._config): - await clear_run( - run_path, - config=self._config, - ingest_writer=self._ingest, - ) - await self._notify(run_path, IngestState.CLEARED) - return IngestState.CLEARED - return IngestState.SYNC_VERIFIED - - case IngestState.CLEARED: - return IngestState.CLEARED - - return current - - # ------------------------------------------------------------------ bootstrap - - async def _bootstrap_initial_ingest(self, loc: _RunLocator) -> None: - """Write the initial ``ingest.json`` payload (state == staging). - - Uses the equipment-side ``creation.json`` for project / kind / - equipment id when available; otherwise uses path-derived defaults. - """ - creation = await self._read_creation_safe(loc.creation_path) - # The path-derived equipment id is authoritative because the staging - # tree mirrors the NAS layout (§13.2). The creation snapshot is - # used only for descriptive metadata (project name, run kind). - equipment_id = loc.equipment_id - equipment = self._equipment_by_id.get(equipment_id) - transport = self._infer_transport(equipment) - run_kind = creation.run_kind if creation is not None else loc.run_kind - project_name = self._project_name_from_creation(creation) or loc.project_name - host = default_host() - run_relative = self._run_relative_path(loc.run_path) - payload = IngestJson( - schema_version=INGEST_JSON_VERSION, - project_name=project_name, - equipment_id=equipment_id, - run_kind=run_kind, - run_path=run_relative, - transport=transport, - current_state=IngestState.STAGING, - history=[ - { - "state": IngestState.STAGING.value, - "at": utc_now_iso(), - "host": host, - }, - ], - ) - await self._ingest.write_ingest(loc.ingest_path, payload) - await self._notify(loc.run_path, IngestState.STAGING) - - async def _read_creation_safe(self, path: Path) -> CreationJson | None: - """Read ``creation.json`` if present and parsable, else ``None``.""" - if not path.exists(): # noqa: ASYNC240 -- one-shot stat - return None - try: - return await self._cache_creation.read_creation_snapshot(path) - except Exception as exc: - _log.warning("creation.json at %s could not be read: %s", path, exc) - return None - - @staticmethod - def _project_name_from_creation(creation: CreationJson | None) -> str | None: - if creation is None: - return None - return creation.lims_project.name_at_creation or creation.lims_project.short_id or None - - def _infer_transport(self, equipment: EquipmentConfig | None) -> OrchestratorTransportType: - """Return the configured staging transport, or a sensible default.""" - if equipment is None or equipment.orchestrator_staging_transport is None: - return OrchestratorTransportType.SMB_MOUNT - return equipment.orchestrator_staging_transport.type - - # ------------------------------------------------------------------ helpers - - async def _advance( - self, - loc: _RunLocator, - *, - next_state: IngestState, - files_received: int | None = None, - bytes_received: int | None = None, - nas_path: str | None = None, - checksum_file: str | None = None, - ) -> None: - """Append the state transition + invoke ``on_state_change`` hook.""" - await self._ingest.append_state_transition( - loc.ingest_path, - next_state, - host=default_host(), - files_received=files_received, - bytes_received=bytes_received, - nas_path=nas_path, - checksum_file=checksum_file, - ) - await self._notify(loc.run_path, next_state) - - async def _notify(self, run_path: Path, state: IngestState) -> None: - """Invoke the optional ``on_state_change`` hook (sync or async).""" - if self._on_state_change is None: - return - result = self._on_state_change(run_path, state) - if asyncio.iscoroutine(result): - await result - - async def _completeness_signal_present(self, loc: _RunLocator) -> bool: - """Return True if the equipment's configured signal is present. - - The check is per-equipment per §13.5: - - * ``sentinel_file`` -- a file with ``equipment.sentinel_filename`` - exists in the run leaf. - * ``manifest`` -- a file with ``equipment.manifest_filename`` - exists AND every file it lists is present with the right size. - - Redesign §3.3: if the equipment isn't in this device's local - registry (received-equipment path), the signal config travels - with the pushed ``creation.json`` ``orchestrator`` block so the - watcher can auto-discover what to look for without a per-equipment - config of its own. - """ - signal_kind, sentinel_filename, manifest_filename = await self._completeness_signal_for(loc) - if signal_kind is None: - return False - match signal_kind: - case CompletenessSignal.SENTINEL_FILE: - if not sentinel_filename: - return False - return (loc.run_path / sentinel_filename).is_file() - case CompletenessSignal.MANIFEST: - if not manifest_filename: - return False - return _manifest_satisfied( - loc.run_path / manifest_filename, - loc.run_path, - ) - return False - - async def _completeness_signal_for( - self, - loc: _RunLocator, - ) -> tuple[CompletenessSignal | None, str | None, str | None]: - """Resolve the completeness-signal triple for ``loc``. - - For owned equipment (``loc.equipment_id`` is in the local - registry), reads from ``EquipmentConfig``. For received equipment - (Redesign §3.3 auto-discovery), falls back to the - ``orchestrator`` block of the pushed ``creation.json`` which - carries the relay-discovery fields. Returns ``(None, None, None)`` - if neither source has the info. - """ - equipment = self._equipment_by_id.get(loc.equipment_id) - if equipment is not None: - return ( - equipment.completeness_signal, - equipment.sentinel_filename, - equipment.manifest_filename, - ) - creation = await self._read_creation_safe(loc.creation_path) - if creation is None or creation.orchestrator is None: - return (None, None, None) - return ( - creation.orchestrator.completeness_signal, - creation.orchestrator.sentinel_filename, - creation.orchestrator.manifest_filename, - ) - - # ------------------------------------------------------------------ scanning - - def _walk_run_leaves(self, staging_root: Path) -> list[Path]: - """Return every ``Run_*`` / ``TestRun_*`` directory under staging_root.""" - return walk_run_leaves(staging_root) - - def _locator_for(self, run_path: Path) -> _RunLocator | None: - """Compute a :class:`_RunLocator` for the run, or None if path is wrong shape.""" - try: - staging_root = Path(self._config.orchestrator.staging_root).resolve() - relative = run_path.resolve().relative_to(staging_root) - except (ValueError, OSError): - return None - parts = relative.parts - equipment_id = parts[0] if parts else "" - # The project name sits at parts[1] and the run leaf is parts[-1]. - project_name = parts[1] if len(parts) >= 2 else "" - run_name = run_path.name - if is_test_run_dir(run_name): - run_kind = RunKind.TEST - elif is_run_dir(run_name): - run_kind = RunKind.EXPERIMENTAL - else: - return None - creation_path = creation_json_path(run_path) - return _RunLocator( - run_path=run_path, - cache_dir=creation_path.parent, - creation_path=creation_path, - ingest_path=ingest_json_path(run_path), - equipment_id=equipment_id, - project_name=project_name, - run_kind=run_kind, - ) - - def _run_relative_path(self, run_path: Path) -> str: - """Return ``run_path`` relative to the staging root (forward-slash).""" - try: - staging_root = Path(self._config.orchestrator.staging_root).resolve() - return run_path.resolve().relative_to(staging_root).as_posix() - except (ValueError, OSError): - return run_path.as_posix() - - @staticmethod - def _count_files_and_bytes(run_path: Path) -> tuple[int, int]: - """Count files + bytes under ``run_path`` excluding the cache dir.""" - return count_files_and_bytes(run_path, exclude_cache=True) - - -def _manifest_satisfied(manifest_path: Path, run_path: Path) -> bool: - """Return True if ``manifest_path`` exists and every listed file is present. - - Manifest format is the spec-implicit ``{"files": [{"path": ..., "size": - ...}]}`` shape. Sizes are compared by exact equality. A manifest with - no ``files`` array is treated as "no files expected" -- which means - the run is complete the moment the manifest itself is on disk. - """ - if not manifest_path.is_file(): - return False - try: - data = read_msgspec_json_raw(manifest_path) - except (msgspec.DecodeError, OSError): - return False - if not isinstance(data, dict): - return False - files = data.get("files", []) - if not isinstance(files, list): - return False - for entry in files: - if not isinstance(entry, dict): - return False - relative = entry.get("path") - size = entry.get("size") - if not isinstance(relative, str) or not relative: - return False - target = run_path / relative - if not target.is_file(): - return False - if isinstance(size, int): - try: - if target.stat().st_size != size: - return False - except OSError: - return False - return True diff --git a/src/exlab_wizard/paths.py b/src/exlab_wizard/paths.py index a29b054..477a683 100644 --- a/src/exlab_wizard/paths.py +++ b/src/exlab_wizard/paths.py @@ -25,7 +25,6 @@ EQUIPMENT_ID_MAX_LENGTH, EQUIPMENT_ID_PATTERN, EQUIPMENT_JSON_NAME, - INGEST_JSON_NAME, PROJECT_NAME_MAX_LENGTH, PROJECT_SHORT_ID_PATTERN, README_FIELDS_JSON_NAME, @@ -59,7 +58,6 @@ "ensure_state_dir", "equipment_json_path", "evaluate_setup_state", - "ingest_json_path", "is_run_dir", "is_test_run_dir", "os_cache_path", @@ -575,11 +573,6 @@ def creation_json_path(run_or_project_dir: Path) -> Path: return cache_dir(run_or_project_dir) / CREATION_JSON_NAME -def ingest_json_path(run_dir: Path) -> Path: - """Return the ``ingest.json`` path under a run directory.""" - return cache_dir(run_dir) / INGEST_JSON_NAME - - def equipment_json_path(equipment_dir: Path) -> Path: """Return the ``equipment.json`` path under an equipment directory.""" return cache_dir(equipment_dir) / EQUIPMENT_JSON_NAME diff --git a/src/exlab_wizard/sync/nas_client.py b/src/exlab_wizard/sync/nas_client.py index d4edba0..64088c3 100644 --- a/src/exlab_wizard/sync/nas_client.py +++ b/src/exlab_wizard/sync/nas_client.py @@ -14,7 +14,7 @@ import asyncio import contextlib -import shutil +import tempfile from collections.abc import Awaitable, Callable from dataclasses import dataclass from datetime import datetime @@ -23,14 +23,15 @@ from exlab_wizard.api.schemas import CreationJson from exlab_wizard.cache.creation_writer import CreationWriter +from exlab_wizard.cache.sync_state_writer import SyncStateWriter from exlab_wizard.config.models import Config, EquipmentConfig, RcloneTransport, RsyncSshTransport from exlab_wizard.constants import ( - CACHE_DIR_NAME, + RunSyncState, SyncHandleState, SyncStatus, ) from exlab_wizard.logging import get_logger -from exlab_wizard.paths import creation_json_path +from exlab_wizard.paths import cache_dir, creation_json_path from exlab_wizard.sync.bandwidth import effective_bandwidth_limit_kibps from exlab_wizard.sync.cleanup import cleanup_interlocks_satisfied from exlab_wizard.sync.pre_sync_gate import is_eligible @@ -39,6 +40,7 @@ SyncJobState, SyncQueue, ) +from exlab_wizard.sync.run_delete import delete_run_files from exlab_wizard.sync.transports import ( TransportError, TransportErrorKind, @@ -61,6 +63,22 @@ _log = get_logger(__name__) +# Job states that count as "done with this subset" for re-enqueue purposes +# (operator-free per-file NAS sync, 2026-05-21). When ``enqueue`` is called +# with a fresh ``files`` list and the run's existing job is in one of these +# states, the row is re-armed in QUEUED with the new subset -- this is how a +# file modified after a prior verify, or queued onto a permanently-failed +# run, gets re-synced. +_TERMINAL_ENQUEUE_STATES: frozenset[SyncJobState] = frozenset( + { + SyncJobState.VERIFIED, + SyncJobState.CLEANUP_ELIGIBLE, + SyncJobState.CLEANED, + SyncJobState.FAILED, + }, +) + + # --------------------------------------------------------------------------- # Public DTOs # --------------------------------------------------------------------------- @@ -114,9 +132,16 @@ def _build_transport_driver(equipment: EquipmentConfig) -> tuple[Any, Callable[. remote_name = transport.rclone_remote remote_path = transport.rclone_remote_path - async def _push_rclone(local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push_rclone( + local: Path, + *, + bwlimit_kibps: int | None, + files_from: Path | None = None, + ) -> TransportResult: target = f"{remote_name}:{remote_path}/{local.name}" - return await rclone_driver.push(local, target, bwlimit_kibps=bwlimit_kibps) + return await rclone_driver.push( + local, target, bwlimit_kibps=bwlimit_kibps, files_from=files_from + ) return rclone_driver, _push_rclone @@ -126,7 +151,12 @@ async def _push_rclone(local: Path, *, bwlimit_kibps: int | None) -> TransportRe ssh_target = transport.ssh_target remote_path_value = transport.remote_path - async def _push_rsync(local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push_rsync( + local: Path, + *, + bwlimit_kibps: int | None, + files_from: Path | None = None, + ) -> TransportResult: target = f"{remote_path_value}/{local.name}" return await rsync_driver.push( local, @@ -134,6 +164,7 @@ async def _push_rsync(local: Path, *, bwlimit_kibps: int | None) -> TransportRes ssh_key, target, bwlimit_kibps=bwlimit_kibps, + files_from=files_from, ) return rsync_driver, _push_rsync @@ -220,6 +251,7 @@ def __init__( queue_db: Path, validator: Validator, cache_creation: CreationWriter, + sync_state_writer: SyncStateWriter | None = None, verifier: Verifier | None = None, worker_poll_interval_s: float = 0.05, push_callable_factory: Callable[[EquipmentConfig], Callable[..., Any]] | None = None, @@ -232,6 +264,7 @@ def __init__( self._queue_db = queue_db self._validator = validator self._cache_creation = cache_creation + self._sync_state_writer = sync_state_writer or SyncStateWriter() self._verifier = verifier or Verifier() self._queue = SyncQueue(queue_db) self._equipment_by_id = {e.id: e for e in config.equipment} @@ -266,14 +299,44 @@ async def close(self) -> None: # ------------------------------------------------------------------ enqueue - async def enqueue(self, run_path: Path) -> SyncJobHandle: + async def enqueue( + self, + run_path: Path, + files: list[str] | None = None, + ) -> SyncJobHandle: """Pre-Sync Gate -> if hard-tier finding without override, mark ``sync_status='blocked_by_validation'``. Otherwise insert a ``QUEUED`` row. + ``files`` (operator-free per-file NAS sync, 2026-05-21) is the + per-file subset of run-relative POSIX paths eligible at enqueue + time; an empty / omitted list means "the whole run". + + The queue holds one row per ``run_path`` (UNIQUE). Re-enqueue + behaviour: + + * existing job in a **terminal** state (``VERIFIED`` / + ``CLEANUP_ELIGIBLE`` / ``CLEANED`` / ``FAILED``) **and** a + non-empty ``files`` list -> reset to ``QUEUED`` carrying the new + subset. This is how a file modified after a prior verify gets + re-synced. + * existing job in a terminal ``VERIFIED`` / ``CLEANUP_ELIGIBLE`` / + ``CLEANED`` state with an **empty** ``files`` list -> falls + through to a no-op: there is no subset to re-sync and a + successfully-verified run is not blindly re-queued. (Only a + terminal ``FAILED`` row with empty ``files`` is re-armed -- the + manual-retry branch below.) + * existing job **active** (``QUEUED`` / ``RUNNING`` / + ``AWAITING_VERIFY``) -> no-op (newly settled files ride the next + sweep). + * existing terminal ``FAILED`` job with no ``files`` -> re-armed + via ``reset_to_queued`` so the manual-retry contract holds. + * no existing job -> insert a ``QUEUED`` row with ``files``. + Returns a :class:`SyncJobHandle`. The handle's ``state`` is either :attr:`SyncHandleState.BLOCKED` or :attr:`SyncHandleState.QUEUED`. """ + files_tuple: tuple[str, ...] = tuple(files or ()) creation_path = creation_json_path(run_path) creation = await self._cache_creation.read_creation_snapshot(creation_path) @@ -294,9 +357,19 @@ async def enqueue(self, run_path: Path) -> SyncJobHandle: equipment_id = self._infer_equipment_id(run_path, creation) existing = await self._queue.get_by_run_path(run_path) if existing is not None: - # Re-enqueueing an existing run is a no-op except for FAILED rows, - # which we re-arm in QUEUED. + if existing.state in _TERMINAL_ENQUEUE_STATES and files_tuple: + # A file modified after a prior verify (or a permanently + # failed run carrying a fresh subset): re-arm the row in + # QUEUED with the new file list. + row = await self._queue.requeue_with_files(existing.id, files_tuple) + self._wake_event.set() + return SyncJobHandle( + job_id=row.id, + state=SyncHandleState.QUEUED, + run_path=str(run_path), + ) if existing.state == SyncJobState.FAILED: + # Manual retry with no fresh subset -- keep the old contract. row = await self._queue.reset_to_queued(existing.id) self._wake_event.set() return SyncJobHandle( @@ -304,6 +377,7 @@ async def enqueue(self, run_path: Path) -> SyncJobHandle: state=SyncHandleState.QUEUED, run_path=str(run_path), ) + # Active job (QUEUED / RUNNING / AWAITING_VERIFY): no-op. return SyncJobHandle( job_id=existing.id, state=SyncHandleState.QUEUED, @@ -314,6 +388,7 @@ async def enqueue(self, run_path: Path) -> SyncJobHandle: run_path=run_path, equipment_id=equipment_id, nas_path=self._compute_nas_path(creation), + files=files_tuple, ) self._wake_event.set() return SyncJobHandle(job_id=row.id, state=SyncHandleState.QUEUED, run_path=str(run_path)) @@ -431,12 +506,28 @@ async def _drive_job(self, job: SyncJobRow) -> None: equipment.transport.bandwidth, now_local=datetime.now() ) + # Per-file NAS sync (2026-05-21): when the job carries a file + # subset, write it to a temp ``--files-from`` list so the transport + # copies only those paths. An empty ``job.files`` keeps the + # whole-directory copy. push = self._build_push(equipment) + files_from_path: Path | None = None try: - result = await push(run_path, bwlimit_kibps=bwlimit) - except TransportError as exc: - await self._queue.record_failure(job.id, error=str(exc), terminal=False) - return + if job.files: + files_from_path = self._write_files_from(job.files) + try: + result = await push( + run_path, + bwlimit_kibps=bwlimit, + files_from=files_from_path, + ) + except TransportError as exc: + await self._queue.record_failure(job.id, error=str(exc), terminal=False) + return + finally: + if files_from_path is not None: + with contextlib.suppress(OSError): + files_from_path.unlink() if not result.ok: await self._handle_push_failure(job, result) @@ -450,9 +541,11 @@ async def _drive_job(self, job: SyncJobRow) -> None: # partial transports cheaply; the remote pass closes the # integrity-in-transit gap and is the reason ``equipment`` flows # in here -- the verifier needs the transport-specific hashsum - # callable. + # callable. When the job carries a file subset, the verify pass is + # scoped to that subset. + include = set(job.files) if job.files else None try: - verify_result = await self._verify_pass(run_path, equipment) + verify_result = await self._verify_pass(run_path, equipment, include=include) except FileNotFoundError: await self._queue.record_failure( job.id, @@ -461,6 +554,14 @@ async def _drive_job(self, job: SyncJobRow) -> None: ) return + # Per-file verify reconciliation (operator-free per-file NAS sync, + # design "Failure handling"): credit every file that verified in + # ``sync_state.json`` -- even when the batch job is otherwise marked + # failed, so a single bad file does not block the good ones. The + # job's overall pass/fail (retry/backoff) is decided below from + # ``verify_result.ok`` exactly as before. + await self._reconcile_synced_files(run_path, job, verify_result) + if not verify_result.ok: # Spec §7.1.5 retry-class routing for verify failures. The # remote hashsum probe may have raised TransportError before @@ -579,7 +680,13 @@ async def _handle_push_failure(self, job: SyncJobRow, result: TransportResult) - # NETWORK / UNKNOWN -> backoff retry. await self._queue.record_failure(job.id, error=kind.value, terminal=False) - async def _verify_pass(self, run_path: Path, equipment: EquipmentConfig) -> VerifyResult: + async def _verify_pass( + self, + run_path: Path, + equipment: EquipmentConfig, + *, + include: set[str] | None = None, + ) -> VerifyResult: """Run one local manifest + verify pass, then probe the remote. The local pass is the cheap pre-check; if the local subtree no @@ -589,6 +696,12 @@ async def _verify_pass(self, run_path: Path, equipment: EquipmentConfig) -> Veri :meth:`Verifier.verify_against_remote` (Backend Spec §7.1.4 -- the integrity-in-transit gap closure). + ``include`` (operator-free per-file NAS sync, 2026-05-21) scopes the + local manifest to a run-relative subset; ``None`` hashes the whole + run. The remote probe still walks the whole run subtree -- the + remote-vs-local comparison is keyed on the (possibly subset) local + manifest, so extra remote keys are simply informational. + A :class:`TransportError` from the hashsum probe is surfaced as a verify failure (``ok=False``) carrying the transport's classified ``error_kind``. The §7.1.4 step-2 contract mandates a remote @@ -599,7 +712,7 @@ async def _verify_pass(self, run_path: Path, equipment: EquipmentConfig) -> Veri NETWORK / UNKNOWN -> backoff, every other case including a missing-binary spawn failure -> single retry then terminal). """ - manifest = await self._verifier.compute_local_manifest(run_path) + manifest = await self._verifier.compute_local_manifest(run_path, include) local_result = await self._verifier.verify_against_local(run_path, manifest) if not local_result.ok: return local_result @@ -616,13 +729,102 @@ async def _verify_pass(self, run_path: Path, equipment: EquipmentConfig) -> Veri ) return self._verifier.verify_against_remote(manifest, remote_manifest) + async def _reconcile_synced_files( + self, + run_path: Path, + job: SyncJobRow, + verify_result: VerifyResult, + ) -> None: + """Credit every individually-verified file in ``sync_state.json``. + + Operator-free per-file NAS sync design ("Failure handling"): a + per-run batch job may verify some files and fail others. Every file + that *did* verify is recorded with its current ``(st_size, + st_mtime_ns)`` ``synced_signature`` and a ``verified_at`` timestamp + -- even when the batch job is otherwise routed to a retry / FAILED + -- so a single bad file does not block crediting the good ones. + + A file counts as verified when it is present in the verify result's + local manifest and absent from both ``mismatched`` and ``missing``. + When the remote probe could not run at all (``error_kind`` set) + nothing is credited -- no file's NAS copy was confirmed. + """ + if verify_result.error_kind is not None: + return + bad = set(verify_result.mismatched) | set(verify_result.missing) + verified_rel = [rel for rel in verify_result.manifest if rel not in bad] + if not verified_rel: + return + verified_at = utc_now_iso() + for rel in verified_rel: + signature = self._file_signature(run_path / rel) + if signature is None: + continue + with contextlib.suppress(Exception): + await self._sync_state_writer.upsert_file( + run_path, + rel, + synced_signature=signature, + verified_at=verified_at, + ) + + @staticmethod + def _file_signature(path: Path) -> tuple[int, int] | None: + """Return the ``(st_size, st_mtime_ns)`` signature for ``path``.""" + try: + stat = path.stat() + except OSError: + return None + return (stat.st_size, stat.st_mtime_ns) + + @staticmethod + def _write_files_from(files: tuple[str, ...]) -> Path: + """Write a transport ``--files-from`` list and return its path. + + One run-relative POSIX path per line. The caller is responsible for + unlinking the temp file once the transport invocation completes. + """ + handle = tempfile.NamedTemporaryFile( # noqa: SIM115 -- caller unlinks + mode="w", + encoding="utf-8", + prefix="exlab-files-from-", + suffix=".txt", + delete=False, + ) + try: + handle.write("\n".join(files) + "\n") + finally: + handle.close() + return Path(handle.name) + async def _maybe_cleanup(self, job_id: str, run_path: Path) -> None: - """Apply the §7.1.6 interlocks; if all pass, run the cleanup.""" + """Apply the §7.1.6 interlocks; if all pass, run the cleanup. + + Operator-free per-file NAS sync design ("Cleanup -- rollup"): with + per-file sync a job reaching ``VERIFIED`` only means *that job's + file subset* verified -- the run may still hold unsynced files from + a later sweep. Cleanup therefore additionally requires the whole-run + ``sync_state.json`` rollup to be ``SYNCED`` (every tracked file + verified); a partially-synced run is left for a later pass. + """ if not self._config.nas_cleanup.enabled: return job = await self._queue.get_by_id(job_id) if job is None or job.state != SyncJobState.VERIFIED: return + + # Whole-run rollup gate: every tracked file must be verified before + # any local deletion. A job's VERIFIED only covers its own subset. + sync_state = await self._sync_state_writer.read(run_path) + rollup = self._sync_state_writer.rollup_state(sync_state) + if rollup != RunSyncState.SYNCED: + _log.debug( + "cleanup deferred: run %s not fully SYNCED (rollup=%s)", + run_path, + rollup.value, + ) + return + creation_path = creation_json_path(run_path) creation: CreationJson | None = None if creation_path.exists(): @@ -643,33 +845,38 @@ async def _maybe_cleanup(self, job_id: str, run_path: Path) -> None: await self._queue.transition(job_id, SyncJobState.CLEANUP_ELIGIBLE) return - # Promote to CLEANUP_ELIGIBLE then perform the deletion. + # Promote to CLEANUP_ELIGIBLE then perform the deletion. Files the + # operator flagged ``keep_local`` survive the sweep. await self._queue.transition(job_id, SyncJobState.CLEANUP_ELIGIBLE) - self._delete_local(run_path) + keep_local = {rel for rel, rec in sync_state.files.items() if rec.keep_local} + self._delete_local(run_path, keep_local) await self._mark_cleaned(run_path) + # Stamp ``cleared_at`` in ``sync_state.json`` so the run rolls up to + # CLEARED. Skipped when the whole-run ``retain_cache=False`` delete + # removed the cache directory along with the data files -- there is + # no surviving record to stamp. + if cache_dir(run_path).exists(): + await self._sync_state_writer.mark_cleared(run_path) await self._queue.transition(job_id, SyncJobState.CLEANED) - def _delete_local(self, run_path: Path) -> None: - """Delete ``run_path`` data files honoring ``retain_cache``. + def _delete_local( + self, + run_path: Path, + keep_local: set[str] | None = None, + ) -> None: + """Delete ``run_path`` data files honoring ``retain_cache`` and ``keep_local``. - With the default ``retain_cache=True`` we keep the - ``.exlab-wizard/`` subtree so the local browse view can still - render the run with a ``cleaned`` badge (§7.1.10). + Thin wrapper over the shared :func:`exlab_wizard.sync.run_delete.delete_run_files` + helper so the automatic cleanup reaper and the operator-facing + ``clear_run_dir`` stay in lockstep: ``keep_local`` files survive, the + ``.exlab-wizard/`` subtree survives (when ``retain_cache``), and + directory symlinks are never descended into or removed. """ - if not run_path.exists(): - return - retain = self._config.nas_cleanup.retain_cache - if retain: - for entry in run_path.iterdir(): - if entry.name == CACHE_DIR_NAME: - continue - if entry.is_dir(): - shutil.rmtree(entry, ignore_errors=True) - else: - with contextlib.suppress(OSError): - entry.unlink() - else: - shutil.rmtree(run_path, ignore_errors=True) + delete_run_files( + run_path, + keep_local=keep_local or set(), + retain_cache=self._config.nas_cleanup.retain_cache, + ) # ----------------------------------------------------------- helpers diff --git a/src/exlab_wizard/sync/queue.py b/src/exlab_wizard/sync/queue.py index bae35c9..b2b7387 100644 --- a/src/exlab_wizard/sync/queue.py +++ b/src/exlab_wizard/sync/queue.py @@ -31,12 +31,19 @@ verify_passes INTEGER NOT NULL DEFAULT 0, verified_at TEXT, enqueued_at TEXT NOT NULL, - nas_path TEXT + nas_path TEXT, + files TEXT NOT NULL DEFAULT '[]' ) + +The ``files`` column holds a JSON array of run-relative POSIX paths -- +the per-file subset eligible at enqueue time (operator-free per-file NAS +sync design, 2026-05-21). An empty list (``'[]'``) means "the whole run", +preserving back-compat for callers that do not pass a file list. """ from __future__ import annotations +import json import uuid from dataclasses import dataclass, replace from datetime import datetime, timedelta @@ -93,7 +100,8 @@ class SyncJobState(StrEnum): verify_passes INTEGER NOT NULL DEFAULT 0, verified_at TEXT, enqueued_at TEXT NOT NULL, - nas_path TEXT + nas_path TEXT, + files TEXT NOT NULL DEFAULT '[]' ) """ @@ -120,6 +128,29 @@ class SyncJobRow: verified_at: str | None = None enqueued_at: str = "" nas_path: str | None = None + files: tuple[str, ...] = () + + +def _decode_files(raw: str | None) -> tuple[str, ...]: + """Decode the ``files`` JSON column into a tuple of run-relative paths. + + A ``NULL`` / empty / ``'[]'`` column yields ``()`` -- the "whole run" + semantics that pre-Phase-4 callers rely on. + """ + if not raw: + return () + try: + decoded = json.loads(raw) + except (ValueError, TypeError): + return () + if not isinstance(decoded, list): + return () + return tuple(str(item) for item in decoded) + + +def _encode_files(files: tuple[str, ...]) -> str: + """Encode a run-relative path tuple into the ``files`` JSON column.""" + return json.dumps(list(files)) def _row_to_job(row: aiosqlite.Row | tuple) -> SyncJobRow: @@ -137,6 +168,7 @@ def _row_to_job(row: aiosqlite.Row | tuple) -> SyncJobRow: verified_at=row[9], enqueued_at=row[10] or "", nas_path=row[11], + files=_decode_files(row[12]), ) @@ -228,9 +260,13 @@ async def insert( equipment_id: str, nas_path: str | None = None, job_id: str | None = None, + files: tuple[str, ...] | list[str] | None = None, ) -> SyncJobRow: """Insert a new ``QUEUED`` row for ``run_path``. + ``files`` is the per-file subset (run-relative POSIX paths) eligible + at enqueue time; an empty / omitted list means "the whole run". + Raises :class:`aiosqlite.IntegrityError` (via the UNIQUE constraint on ``run_path``) if a row already exists for the same path. """ @@ -242,14 +278,15 @@ async def insert( state=SyncJobState.QUEUED, enqueued_at=utc_now_iso(), nas_path=nas_path, + files=tuple(files or ()), ) await conn.execute( """ INSERT INTO jobs ( id, run_path, equipment_id, state, attempts, last_attempt_at, next_attempt_at, last_error, - verify_passes, verified_at, enqueued_at, nas_path - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + verify_passes, verified_at, enqueued_at, nas_path, files + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( row.id, @@ -264,6 +301,7 @@ async def insert( row.verified_at, row.enqueued_at, row.nas_path, + _encode_files(row.files), ), ) await conn.commit() @@ -459,6 +497,40 @@ async def reset_to_queued(self, job_id: str) -> SyncJobRow: await conn.commit() return await self.get_by_id(job_id) # type: ignore[return-value] + async def requeue_with_files( + self, + job_id: str, + files: tuple[str, ...] | list[str], + ) -> SyncJobRow: + """Reset a terminal job back to ``QUEUED`` carrying a new ``files`` list. + + Operator-free per-file NAS sync design (2026-05-21): a file modified + after a prior verify becomes re-eligible. When the poller observes + such a file it re-arms the run's terminal job (``VERIFIED`` / + ``CLEANUP_ELIGIBLE`` / ``CLEANED`` / ``FAILED``) with the freshly + eligible subset. ``attempts`` / ``last_error`` / ``verify_passes`` + are cleared so the backoff schedule and verify counter start fresh. + """ + await self._require_job(job_id) + conn = self._require_conn() + await conn.execute( + """ + UPDATE jobs SET + state = ?, + attempts = 0, + last_attempt_at = NULL, + next_attempt_at = NULL, + last_error = NULL, + verify_passes = 0, + verified_at = NULL, + files = ? + WHERE id = ? + """, + (SyncJobState.QUEUED.value, _encode_files(tuple(files)), job_id), + ) + await conn.commit() + return await self.get_by_id(job_id) # type: ignore[return-value] + async def delete(self, job_id: str) -> None: """Remove a job row entirely. Used by the cleanup reaper after CLEANED.""" conn = self._require_conn() diff --git a/src/exlab_wizard/sync/run_delete.py b/src/exlab_wizard/sync/run_delete.py new file mode 100644 index 0000000..b747522 --- /dev/null +++ b/src/exlab_wizard/sync/run_delete.py @@ -0,0 +1,116 @@ +"""Keep-local-aware, symlink-safe deletion of a run's staging copy. + +Operator-free per-file NAS sync design (2026-05-21). Both the automatic +cleanup reaper (:meth:`exlab_wizard.sync.nas_client.NASSyncClient._delete_local`) +and the operator-facing "Clear" actions +(:func:`exlab_wizard.orchestrator.staging_clear.clear_run_dir`) must delete a +run's staging copy with **identical** semantics: + +* files flagged ``keep_local`` in ``sync_state.json`` survive (the spec's + keep-local guarantee: "excluded from cleanup deletion"); +* the ``.exlab-wizard/`` metadata subtree survives so a cleared run still + renders its files as "On NAS" tombstones; +* directory **symlinks are never descended into or removed** -- a staged run + containing a symlink to an external directory must not have files deleted + outside the run tree. + +This module is the single shared implementation. It is a pure filesystem +helper -- no ``SyncStateWriter`` / ``api`` / ``orchestrator`` imports -- so it +can be imported from anywhere without circular-import risk. The callers own +reading the ``keep_local`` set and stamping ``cleared_at``. +""" + +from __future__ import annotations + +import contextlib +import os +import shutil +from pathlib import Path + +from exlab_wizard.constants import CACHE_DIR_NAME +from exlab_wizard.logging import get_logger + +__all__ = ["delete_run_files"] + +_log = get_logger(__name__) + + +def delete_run_files( + run_path: Path, + *, + keep_local: set[str], + retain_cache: bool, +) -> None: + """Delete ``run_path`` data files honoring ``retain_cache`` and ``keep_local``. + + ``keep_local`` is a set of run-relative POSIX paths (possibly nested) that + must survive the sweep. ``retain_cache`` keeps the ``.exlab-wizard/`` + subtree when ``True``. + + The whole-run ``shutil.rmtree`` fast path is used only when + ``retain_cache`` is ``False`` **and** there are no ``keep_local`` files; + any kept file (or a retained cache) forces the per-file walk so it + survives. The walk does not follow directory symlinks -- a symlinked + directory inside the run is left entirely untouched (neither its contents + deleted nor the link removed). + + Idempotent: a missing ``run_path`` is a no-op. + """ + if not run_path.exists(): + return + if not retain_cache and not keep_local: + # No retained files and no cache to preserve: drop the whole run. + # ``rmtree`` does not follow the top-level dir if it is itself a + # symlink (it raises) -- a run dir is always a real directory here. + shutil.rmtree(run_path, ignore_errors=True) + return + + # Per-file walk: delete every file that is neither under + # ``.exlab-wizard/`` nor flagged ``keep_local``. ``followlinks=False`` + # (the os.walk default, made explicit) keeps the walk inside the run + # tree -- a symlinked subdirectory is yielded as a name but never + # descended into, so its target's contents are never touched. + for dirpath, dirnames, filenames in os.walk(run_path, followlinks=False): + current = Path(dirpath) + # Do not descend into the cache subtree or any symlinked directory. + dirnames[:] = [ + d for d in dirnames if d != CACHE_DIR_NAME and not (current / d).is_symlink() + ] + rel_dir = current.relative_to(run_path) + if rel_dir.parts and rel_dir.parts[0] == CACHE_DIR_NAME: + continue + for name in filenames: + entry = current / name + rel = (rel_dir / name).as_posix() + if rel in keep_local: + continue + # A file that is itself a symlink: unlink the link only (never + # the target). ``unlink`` does exactly that. + with contextlib.suppress(OSError): + entry.unlink() + _prune_empty_dirs(run_path) + + +def _prune_empty_dirs(run_path: Path) -> None: + """Remove now-empty real directories under ``run_path`` (deepest first). + + The run directory itself, the ``.exlab-wizard/`` cache subtree, and any + symlinked directory are never removed. A real directory left empty by the + delete walk is pruned so cleanup leaves only retained files and metadata. + """ + cache_dir_path = run_path / CACHE_DIR_NAME + real_dirs: list[Path] = [] + for dirpath, dirnames, _filenames in os.walk(run_path, followlinks=False): + current = Path(dirpath) + # Prune symlinked directories from the descent so we never rmdir one. + dirnames[:] = [d for d in dirnames if not (current / d).is_symlink()] + if current != run_path: + real_dirs.append(current) + # Deepest first so a parent emptied by pruning its children is itself + # prunable in the same pass. + for directory in sorted(real_dirs, key=lambda p: len(p.parts), reverse=True): + if directory == cache_dir_path or cache_dir_path in directory.parents: + continue + with contextlib.suppress(OSError): + if not any(directory.iterdir()): + directory.rmdir() diff --git a/src/exlab_wizard/sync/transports/rclone.py b/src/exlab_wizard/sync/transports/rclone.py index 40c4dcf..0cd2097 100644 --- a/src/exlab_wizard/sync/transports/rclone.py +++ b/src/exlab_wizard/sync/transports/rclone.py @@ -70,6 +70,7 @@ async def push( remote: str, *, bwlimit_kibps: int | None = None, + files_from: Path | None = None, ) -> TransportResult: """Run ``rclone copy --checksum`` from ``local`` to ``remote``. @@ -77,6 +78,11 @@ async def push( rclone spec. ``bwlimit_kibps`` (KiB/s) is forwarded as ``--bwlimit K`` when set. + ``files_from`` (operator-free per-file NAS sync, 2026-05-21), when + set, is a path to a text file listing run-relative paths to copy -- + forwarded as ``--files-from `` so only that subset transfers. + ``None`` keeps the whole-directory copy behaviour. + Returns a :class:`TransportResult` describing the outcome. A process-spawn failure (binary missing) raises :class:`TransportError` because no retry will help -- the lab @@ -85,6 +91,8 @@ async def push( cmd: list[str] = [self._binary, "copy", "--checksum"] if bwlimit_kibps is not None and bwlimit_kibps > 0: cmd.extend(["--bwlimit", f"{bwlimit_kibps}K"]) + if files_from is not None: + cmd.extend(["--files-from", str(files_from)]) cmd.extend([str(local), remote]) _log.debug("rclone cmd: %s", shlex.join(cmd)) diff --git a/src/exlab_wizard/sync/transports/rsync_ssh.py b/src/exlab_wizard/sync/transports/rsync_ssh.py index 3650363..11b8375 100644 --- a/src/exlab_wizard/sync/transports/rsync_ssh.py +++ b/src/exlab_wizard/sync/transports/rsync_ssh.py @@ -89,6 +89,7 @@ async def push( remote_path: str, *, bwlimit_kibps: int | None = None, + files_from: Path | None = None, ) -> TransportResult: """Run ``rsync -avz --checksum`` from ``local`` to ``ssh_target:remote_path``. @@ -96,6 +97,11 @@ async def push( via ``-e 'ssh -i -o BatchMode=yes'`` so the driver never prompts for a password. + ``files_from`` (operator-free per-file NAS sync, 2026-05-21), when + set, is a path to a text file listing run-relative paths to copy -- + forwarded as ``--files-from `` so only that subset transfers. + ``None`` keeps the whole-directory copy behaviour. + Returns a :class:`TransportResult`. Raises :class:`TransportError` when the rsync binary is missing (no retry will help). """ @@ -110,6 +116,12 @@ async def push( ] if bwlimit_kibps is not None and bwlimit_kibps > 0: cmd.append(f"--bwlimit={bwlimit_kibps}") + if files_from is not None: + # The paths inside the --files-from file are interpreted by + # rsync as relative to the source dir argument (``local`` + # below), which is exactly the run-relative POSIX layout the + # caller writes -- so no path rewriting is needed here. + cmd.append(f"--files-from={files_from}") cmd.append(str(local)) cmd.append(f"{ssh_target}:{remote_path}") _log.debug("rsync cmd: %s", shlex.join(cmd)) diff --git a/src/exlab_wizard/sync/verifier.py b/src/exlab_wizard/sync/verifier.py index 094eeab..dec2962 100644 --- a/src/exlab_wizard/sync/verifier.py +++ b/src/exlab_wizard/sync/verifier.py @@ -63,17 +63,24 @@ class VerifyResult: error_kind: TransportErrorKind | None = None -def _iter_files(run_path: Path) -> list[Path]: +def _iter_files(run_path: Path, include: set[str] | None = None) -> list[Path]: """Return every regular file under ``run_path`` (depth-first). Uses ``Path.rglob('*')`` and filters to regular files. The manifest format is independent of walk order, but we sort the result by relative path so the manifest file is reproducible byte-for-byte. + + ``include`` (operator-free per-file NAS sync, 2026-05-21), when set, is + a set of run-relative POSIX paths; only files whose relative path is in + the set are returned. ``None`` walks the whole subtree (unchanged). """ files: list[Path] = [] for path in run_path.rglob("*"): - if path.is_file(): - files.append(path) + if not path.is_file(): + continue + if include is not None and path.relative_to(run_path).as_posix() not in include: + continue + files.append(path) return files @@ -145,29 +152,46 @@ def parse_manifest(text: str) -> dict[str, str]: class Verifier: """SHA-256 verifier. Backend Spec §7.1.4.""" - async def compute_local_manifest(self, run_path: Path) -> dict[str, str]: + async def compute_local_manifest( + self, + run_path: Path, + include: set[str] | None = None, + ) -> dict[str, str]: """Walk ``run_path`` and compute a SHA-256 per file. Writes the manifest to ``run_path/.exlab-wizard/checksums.sha256`` as a side-effect (the §7.1.4 contract). Files inside the ``.exlab-wizard/`` cache subtree are excluded so the manifest does not record its own hash. + + ``include`` (operator-free per-file NAS sync, 2026-05-21), when set, + is a set of run-relative POSIX paths; only those files are hashed -- + used to verify a per-file sync subset. ``None`` hashes the whole + subtree (unchanged). + + The side-effect ``checksums.sha256`` write is skipped for a subset + (``include is not None``): only a whole-run manifest is durable -- + persisting a partial manifest would clobber the run's checksum file + with an incomplete record. """ if not run_path.exists() or not run_path.is_dir(): # noqa: ASYNC240 -- one-shot stat msg = f"run_path does not exist or is not a directory: {run_path}" raise FileNotFoundError(msg) manifest: dict[str, str] = {} - for file_path in _iter_files(run_path): + for file_path in _iter_files(run_path, include): rel = file_path.relative_to(run_path) if _is_inside_cache_dir(rel): continue manifest[str(rel.as_posix())] = await _compute_sha256(file_path) - # Persist to .exlab-wizard/checksums.sha256. - paths.cache_dir(run_path).mkdir(parents=True, exist_ok=True) - checksums_path = run_path / CHECKSUMS_RELATIVE - atomic_write_bytes(checksums_path, format_manifest(manifest).encode("utf-8")) + # Persist to .exlab-wizard/checksums.sha256 -- whole-run only. A + # subset pass must not overwrite the run's checksum file with a + # partial manifest. + if include is None: + paths.cache_dir(run_path).mkdir(parents=True, exist_ok=True) + checksums_path = run_path / CHECKSUMS_RELATIVE + atomic_write_bytes(checksums_path, format_manifest(manifest).encode("utf-8")) return manifest async def verify_against_local(self, run_path: Path, manifest: dict[str, str]) -> VerifyResult: diff --git a/src/exlab_wizard/tray/dependencies.py b/src/exlab_wizard/tray/dependencies.py index b0f1c73..e196f01 100644 --- a/src/exlab_wizard/tray/dependencies.py +++ b/src/exlab_wizard/tray/dependencies.py @@ -10,17 +10,17 @@ failure logs WARN"). The order matters: validator depends on the cache writers, controller -composes validator + plugin host + template engine + cache writers, -staging watcher depends on the ingest writer + a NAS-sync stub. We -construct upstream pieces first and pass them into downstream -constructors; any upstream failure short-circuits the chain so a None -upstream produces a None downstream rather than a partially-constructed -object. +composes validator + plugin host + template engine + cache writers, the +quiescence poller depends on the NAS-sync queue. We construct upstream +pieces first and pass them into downstream constructors; any upstream +failure short-circuits the chain so a None upstream produces a None +downstream rather than a partially-constructed object. """ from __future__ import annotations import contextlib +import os from pathlib import Path from typing import Any @@ -62,7 +62,6 @@ def build_production_dependencies(state_dir: Path) -> AppDependencies: cache_equipment = _try("cache_equipment", _build_equipment_writer) template_engine = _try("template_engine", _build_template_engine) deps.plugin_host = _try("plugin_host", _build_plugin_host, deps.config) - deps.ingest_writer = _try("ingest_writer", _build_ingest_writer) deps.controller = _try( "controller", @@ -94,16 +93,28 @@ def build_production_dependencies(state_dir: Path) -> AppDependencies: deps.lims_reachable = True deps.lims_probe = _make_lims_probe(deps) - deps.nas_sync = _try("nas_sync", _build_nas_sync, deps.config, state_dir) + # ``sync_state_writer`` must precede ``nas_sync`` -- the NASSyncClient + # takes it as a constructor dependency for per-file verify + # reconciliation (operator-free per-file NAS sync, 2026-05-21). + deps.sync_state_writer = _try("sync_state_writer", _build_sync_state_writer) + + deps.nas_sync = _try( + "nas_sync", + _build_nas_sync, + deps.config, + state_dir, + validator, + deps.cache_creation, + deps.sync_state_writer, + ) deps.nas_sync_snapshot = _make_nas_sync_snapshot(deps) - deps.staging_watcher = _try( - "staging_watcher", - _build_staging_watcher, + deps.quiescence_poller = _try( + "quiescence_poller", + _build_quiescence_poller, config=deps.config, - ingest_writer=deps.ingest_writer, nas_sync=deps.nas_sync, - cache_creation=deps.cache_creation, + sync_state_writer=deps.sync_state_writer, ) deps.autostart_toggle = _make_autostart_toggle() @@ -192,12 +203,6 @@ def _build_equipment_writer() -> Any: return EquipmentCacheWriter() -def _build_ingest_writer() -> Any: - from exlab_wizard.cache.ingest_writer import IngestWriter - - return IngestWriter() - - def _build_template_engine() -> Any: from exlab_wizard.template.copier_driver import TemplateEngine @@ -252,10 +257,27 @@ def _build_controller( ) +# Env var supplying the master passphrase for the encrypted-at-rest +# secret store. Read only when the OS keyring is unavailable; never +# committed -- callers export it at launch on keyring-less hosts. +_SECRET_PASSPHRASE_ENV = "EXLAB_WIZARD_SECRET_PASSPHRASE" + + def _build_keyring_store(state_dir: Path) -> Any: + """Build the LIMS/NAS secret store with an optional fallback passphrase. + + When the OS keyring backend is unavailable, :class:`KeyringStore` + falls back to an encrypted-at-rest file keyed by a master passphrase + (Backend Spec §7.4.4). That passphrase is read from + ``EXLAB_WIZARD_SECRET_PASSPHRASE``; when the variable is unset no + provider is wired and the fallback stays disabled -- the historical + behaviour, so keyring-equipped hosts are unaffected. + """ from exlab_wizard.lims.keyring_store import KeyringStore - return KeyringStore(state_dir=state_dir) + passphrase = os.environ.get(_SECRET_PASSPHRASE_ENV) + provider = (lambda: passphrase) if passphrase else None + return KeyringStore(state_dir=state_dir, passphrase_provider=provider) def _lims_keyring_password(keyring_store: Any) -> str | None: @@ -323,14 +345,44 @@ async def _probe(_body: Any = None) -> dict[str, Any]: return _probe -def _build_nas_sync(config: Any, state_dir: Path) -> Any: +def _build_nas_sync( + config: Any, + state_dir: Path, + validator: Any, + cache_creation: Any, + sync_state_writer: Any, +) -> Any: + """Build the :class:`NASSyncClient` -- the public NAS-sync surface. + + The client wires the durable queue, the transport drivers, the + verifier, the Pre-Sync Gate, and (operator-free per-file NAS sync, + 2026-05-21) the ``sync_state.json`` writer used for per-file verify + reconciliation. ``verifier`` and the transport / hashsum factories + default correctly inside ``NASSyncClient`` so they are not passed. + + The poller and the force-sync route call ``enqueue`` / ``status`` on + this object; returning a bare ``SyncQueue`` (which has neither) would + leave the poller sweep silently dead. + """ if config is None: msg = "NAS sync requires a loaded config" raise RuntimeError(msg) - from exlab_wizard.sync.queue import SyncQueue + if validator is None: + msg = "NAS sync requires a validator" + raise RuntimeError(msg) + if cache_creation is None: + msg = "NAS sync requires a creation-cache writer" + raise RuntimeError(msg) + from exlab_wizard.sync.nas_client import NASSyncClient db_path = state_dir / "sync_queue.sqlite" - return SyncQueue(db_path) + return NASSyncClient( + config=config, + queue_db=db_path, + validator=validator, + cache_creation=cache_creation, + sync_state_writer=sync_state_writer, + ) def _make_nas_sync_snapshot(deps: AppDependencies) -> Any: @@ -345,27 +397,48 @@ def _snapshot() -> dict[str, Any]: return _snapshot -def _build_staging_watcher( +def _build_sync_state_writer() -> Any: + """Build the orchestrator-only ``sync_state.json`` writer. + + Operator-free per-file NAS sync design (2026-05-21): the quiescence + poller reads ``sync_state.json`` to skip already-synced files and the + NAS-sync client writes per-file verify reconciliation into it. + """ + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + return SyncStateWriter() + + +def _build_quiescence_poller( *, config: Any, - ingest_writer: Any, nas_sync: Any, - cache_creation: Any, + sync_state_writer: Any, ) -> Any: - # Redesign §3.1: the staging watcher boots whenever staging_root is - # configured; the legacy enabled toggle is gone. - if config is None or not config.orchestrator.staging_root: + # Operator-free per-file NAS sync design (2026-05-21): the quiescence + # poller boots whenever a staging_root is configured OR any equipment + # is in ``nas`` sync mode -- it is the single auto-sync trigger for + # both orchestrator-staged and nas-mode runs. + if config is None: return None - if ingest_writer is None or nas_sync is None or cache_creation is None: - msg = "staging watcher requires ingest_writer + nas_sync + cache_creation" + from exlab_wizard.constants import SyncMode + + has_staging_root = bool(config.orchestrator.staging_root) + has_nas_equipment = any(eq.sync_mode == SyncMode.NAS for eq in config.equipment) + if not (has_staging_root or has_nas_equipment): + return None + if nas_sync is None: + msg = "quiescence poller requires nas_sync" raise RuntimeError(msg) - from exlab_wizard.orchestrator.staging_watcher import StagingWatcher + if sync_state_writer is None: + msg = "quiescence poller requires sync_state_writer" + raise RuntimeError(msg) + from exlab_wizard.orchestrator.quiescence_poller import QuiescenceSyncPoller - return StagingWatcher( + return QuiescenceSyncPoller( config=config, - ingest_writer=ingest_writer, nas_sync=nas_sync, - cache_creation=cache_creation, + sync_state_writer=sync_state_writer, ) diff --git a/src/exlab_wizard/tray/main.py b/src/exlab_wizard/tray/main.py index 7e4afed..16b65f8 100644 --- a/src/exlab_wizard/tray/main.py +++ b/src/exlab_wizard/tray/main.py @@ -297,8 +297,6 @@ def _bootstrap_test_config(config_path: Path, *, include_samples: bool) -> None: "label": "Test Rig", "local_root": str(sandbox / "local" / "TESTRIG"), "nas_root": str(sandbox / "nas" / "TESTRIG"), - "completeness_signal": "sentinel_file", - "sentinel_filename": "done.flag", "sync_mode": "nas", "transport": { "type": "rclone", diff --git a/src/exlab_wizard/ui/components/file_list.py b/src/exlab_wizard/ui/components/file_list.py index e777197..9020dfa 100644 --- a/src/exlab_wizard/ui/components/file_list.py +++ b/src/exlab_wizard/ui/components/file_list.py @@ -10,8 +10,13 @@ caller decides when to invoke and when to stop the underlying poll. Right-click context menu (Redesign §4.3 / decision 6A): selecting a row -opens a menu with **Open in OS** and **Copy path** actions; the right -metadata pane is NOT driven by file-list selection. +opens a menu with **Open in OS**, **Copy path**, and **Keep local** +actions; the right metadata pane is NOT driven by file-list selection. + +Operator-free per-file NAS sync design (2026-05-21): a row can be a +**tombstone** -- a file present in the run's ``sync_state.json`` but +absent on disk (an "On NAS" cleared-run file). A tombstone is not +openable. A ``keep_local`` file carries a small "kept local" badge. """ from __future__ import annotations @@ -20,16 +25,24 @@ from dataclasses import dataclass, field from typing import Any +from exlab_wizard.ui.components.sync_status_icon import STATUS_ON_NAS from exlab_wizard.ui.pages.staging import format_bytes # Action discriminators consumed by the on_context_menu callback. FILE_CONTEXT_OPEN = "open_in_os" FILE_CONTEXT_COPY_PATH = "copy_path" +FILE_CONTEXT_KEEP_LOCAL = "keep_local" @dataclass(frozen=True) class FileListEntry: - """One row in the centre-pane file list.""" + """One row in the centre-pane file list. + + ``keep_local`` mirrors the file's ``sync_state.json`` keep-local flag + (excluded from cleanup deletion). ``tombstone`` marks an "On NAS" + row -- a file recorded in ``sync_state.json`` but absent on disk; + such a row shows the ``on_nas`` icon and is not openable. + """ name: str path: str @@ -37,6 +50,8 @@ class FileListEntry: size_bytes: int | None = None modified_iso: str | None = None sync_status: str | None = None + keep_local: bool = False + tombstone: bool = False @dataclass @@ -83,6 +98,8 @@ def diff_file_lists( before.size_bytes != after.size_bytes or before.modified_iso != after.modified_iso or before.sync_status != after.sync_status + or before.keep_local != after.keep_local + or before.tombstone != after.tombstone ): modified.append(path) return FileListDiff( @@ -146,14 +163,28 @@ def _render_row( highlight = "background: var(--color-highlight); " if is_new else "" size_text = "-" if entry.size_bytes is None else format_bytes(int(entry.size_bytes)) modified_text = entry.modified_iso or "-" - sync_text = entry.sync_status or "-" + sync_text = entry.sync_status or (STATUS_ON_NAS if entry.tombstone else "-") + # A tombstone ("On NAS") row is dimmed -- the local copy is gone. + row_style = f"{highlight}border-bottom: 1px solid var(--color-rule);" + if entry.tombstone: + row_style += " opacity: 0.65;" + keep_local_attr = ' data-keep-local="true"' if entry.keep_local else "" + tombstone_attr = ' data-tombstone="true"' if entry.tombstone else "" with ( ui.element("tr") - .style(f"{highlight}border-bottom: 1px solid var(--color-rule);") - .props(f'data-testid="file-list-row" data-path="{entry.path}"') + .style(row_style) + .props( + f'data-testid="file-list-row" data-path="{entry.path}"{keep_local_attr}{tombstone_attr}' + ) ): with ui.element("td").classes("p-2").style("font-weight: 500;"): ui.label(entry.name) + if entry.keep_local: + ui.label("kept local").props('data-testid="file-keep-local-badge"').style( + "display: inline-block; margin-left: 0.4rem; padding: 0 0.35rem; " + "font-size: var(--text-xs); border-radius: var(--radius-sm); " + "background: var(--color-highlight); color: var(--color-muted);" + ) with ui.element("td").classes("p-2 text-right"): ui.label(size_text) with ui.element("td").classes("p-2"): @@ -164,14 +195,22 @@ def _render_row( with ui.context_menu().props( f'data-testid="file-context-menu" data-path="{entry.path}"' ): - ui.menu_item("Open in OS").props('data-testid="file-context-open-in-os"').on( - "click", - lambda _evt, e=entry: on_context_menu(e, FILE_CONTEXT_OPEN), - ) + # A tombstone has no local copy -- "Open in OS" would + # fail, so it is omitted for tombstone rows. + if not entry.tombstone: + ui.menu_item("Open in OS").props('data-testid="file-context-open-in-os"').on( + "click", + lambda _evt, e=entry: on_context_menu(e, FILE_CONTEXT_OPEN), + ) ui.menu_item("Copy path").props('data-testid="file-context-copy-path"').on( "click", lambda _evt, e=entry: on_context_menu(e, FILE_CONTEXT_COPY_PATH), ) + keep_local_label = "Don't keep local" if entry.keep_local else "Keep local" + ui.menu_item(keep_local_label).props('data-testid="file-context-keep-local"').on( + "click", + lambda _evt, e=entry: on_context_menu(e, FILE_CONTEXT_KEEP_LOCAL), + ) if on_double_click is not None: # NiceGUI doesn't expose row-level dblclick easily; the caller is diff --git a/src/exlab_wizard/ui/components/metadata_pane.py b/src/exlab_wizard/ui/components/metadata_pane.py index 48770ea..bffd2d3 100644 --- a/src/exlab_wizard/ui/components/metadata_pane.py +++ b/src/exlab_wizard/ui/components/metadata_pane.py @@ -105,7 +105,6 @@ def _render_equipment( _kv("Sync mode", payload.get("sync_mode")) _kv("Local root", payload.get("local_root")) _kv("NAS root", payload.get("nas_root")) - _kv("Completeness signal", payload.get("completeness_signal")) if payload.get("sync_mode") == "stage": ui.label( "Stage mode: this device pushes runs to a connected PC's staging " diff --git a/src/exlab_wizard/ui/components/sync_status_icon.py b/src/exlab_wizard/ui/components/sync_status_icon.py index 67fdfb0..8762ec4 100644 --- a/src/exlab_wizard/ui/components/sync_status_icon.py +++ b/src/exlab_wizard/ui/components/sync_status_icon.py @@ -1,15 +1,23 @@ """Sync-status icon component (Frontend Spec §3.2, §10.5.1). -Seven distinct visual states with a fixed color mapping: +Distinct visual states with a fixed color mapping: * ``pending`` -- ``--color-muted`` +* ``acquiring`` -- ``--color-muted`` * ``retrying`` (with N/M) -- ``--color-info`` +* ``syncing`` -- ``--color-info`` * ``synced`` -- ``--color-success`` * ``cleaned`` -- ``--color-success`` +* ``on_nas`` -- ``--color-muted`` * ``failed`` -- ``--color-danger`` * ``blocked_by_validation`` -- ``--color-warning`` * ``override_active`` -- ``--color-info`` +The ``acquiring`` / ``syncing`` / ``on_nas`` states are the per-file GUI +display states from the operator-free per-file NAS sync design +(2026-05-21): a file still settling, a file mid-transfer, and an "On +NAS" tombstone whose local copy has been cleared. + The component returns a dict suitable for a NiceGUI icon factory; the layout (icon + optional ``(N/M)`` retry counter) is the caller's concern so the icon can be embedded in a tree row, a detail-pane title bar, or a @@ -34,6 +42,13 @@ STATUS_RETRYING: Final[str] = "retrying" STATUS_OVERRIDE: Final[str] = "override_active" +# Per-file GUI display states (operator-free per-file NAS sync design, +# 2026-05-21). These mirror the discriminators emitted by +# ``api.routers.browse._file_state_from_record``. +STATUS_ACQUIRING: Final[str] = "acquiring" +STATUS_SYNCING: Final[str] = "syncing" +STATUS_ON_NAS: Final[str] = "on_nas" + _STATUS_TO_PROPS: dict[str, dict[str, str]] = { SyncStatus.PENDING.value: { @@ -41,11 +56,26 @@ "color_var": "--color-muted", "tooltip": "Queued for sync", }, + STATUS_ACQUIRING: { + "icon_name": "edit_note", + "color_var": "--color-muted", + "tooltip": "Acquiring -- new file, still settling", + }, STATUS_RETRYING: { "icon_name": "history", "color_var": "--color-info", "tooltip": "Retrying with backoff", }, + STATUS_SYNCING: { + "icon_name": "sync", + "color_var": "--color-info", + "tooltip": "Syncing -- settled, transferring to NAS", + }, + STATUS_ON_NAS: { + "icon_name": "cloud", + "color_var": "--color-muted", + "tooltip": "On NAS -- local copy cleared, data on NAS only", + }, SyncStatus.SYNCED.value: { "icon_name": "check_circle", "color_var": "--color-success", diff --git a/src/exlab_wizard/ui/components/tree.py b/src/exlab_wizard/ui/components/tree.py index 762c234..c3392b5 100644 --- a/src/exlab_wizard/ui/components/tree.py +++ b/src/exlab_wizard/ui/components/tree.py @@ -11,11 +11,15 @@ Run rows also carry a small **sync icon** to the left of the label: -* ``sync_local.svg`` -- run data is still on local disk (any sync - status other than ``cleaned``). -* ``sync_cloud.svg`` -- run has been synced, verified, and locally - cleaned (``sync_status == "cleaned"``); only the ``.exlab-wizard/`` - cache subtree remains on disk (§7.1.10). +* ``sync_local.svg`` -- run data is still on local disk (rollup + ``syncing`` / ``synced``, any state other than ``cleared``). +* ``sync_cloud.svg`` -- the run's staging copy has been cleared + (rollup ``cleared``); only the ``.exlab-wizard/`` cache subtree + remains on disk (§7.1.10). + +The run-node rollup is derived from the run's ``sync_state.json`` by the +browse router (operator-free per-file NAS sync design, 2026-05-21); +``RunNode.sync_status`` carries a :class:`RunSyncState` value. ``.exlab-wizard/`` folders are hidden by default (Frontend §13.1) and hidden filtering is the caller's concern. @@ -31,7 +35,7 @@ from dataclasses import dataclass, field from typing import Any -from exlab_wizard.constants.enums import RunKind, SyncStatus, TreeProjectStatus +from exlab_wizard.constants.enums import RunKind, RunSyncState, TreeProjectStatus from exlab_wizard.logging import get_logger _log = get_logger(__name__) @@ -232,13 +236,14 @@ def _sync_icon_url(node: TreeNode) -> str | None: """Return the per-row sync-icon URL, or ``None`` for non-run rows. Run rows get one of the two ``/assets/sync_*.svg`` URLs depending on - whether the run has been locally cleaned (``sync_cloud.svg``) or - still has data on disk (``sync_local.svg``). Equipment / project - rows render unchanged. + the derived run rollup: a ``cleared`` run (staging copy cleaned, data + on NAS only) gets ``sync_cloud.svg``; a ``syncing`` / ``synced`` run + still has data on disk and gets ``sync_local.svg``. Equipment / + project rows render unchanged. """ if node.kind not in _RUN_KINDS: return None - if node.sync_status == SyncStatus.CLEANED.value: + if node.sync_status == RunSyncState.CLEARED.value: return SYNC_ICON_CLOUD_URL return SYNC_ICON_LOCAL_URL diff --git a/src/exlab_wizard/ui/equipment_form.py b/src/exlab_wizard/ui/equipment_form.py index cab1ee6..969fad2 100644 --- a/src/exlab_wizard/ui/equipment_form.py +++ b/src/exlab_wizard/ui/equipment_form.py @@ -19,7 +19,6 @@ RsyncSshTransport, ) from exlab_wizard.constants import ( - CompletenessSignal, OrchestratorTransportType, SyncMode, ) @@ -33,9 +32,6 @@ def build_equipment_config( label: str, local_root: str, nas_root: str, - completeness_signal: str, - sentinel_filename: str, - manifest_filename: str, sync_mode: str = "nas", # NAS transport fields (when sync_mode == "nas") transport_type: str = "rclone", @@ -58,7 +54,6 @@ def build_equipment_config( Pydantic validation enforces the exclusivity rule. """ mode = SyncMode(sync_mode) - signal = CompletenessSignal(completeness_signal) transport: RcloneTransport | RsyncSshTransport | None = None orch_staging: OrchestratorStagingTransport | None = None @@ -89,15 +84,6 @@ def build_equipment_config( label=label.strip(), local_root=local_root.strip(), nas_root=nas_root.strip(), - completeness_signal=signal, - sentinel_filename=( - sentinel_filename.strip() or None - if signal is CompletenessSignal.SENTINEL_FILE - else None - ), - manifest_filename=( - manifest_filename.strip() or None if signal is CompletenessSignal.MANIFEST else None - ), sync_mode=mode, transport=transport, orchestrator_staging_transport=orch_staging, diff --git a/src/exlab_wizard/ui/mount.py b/src/exlab_wizard/ui/mount.py index cd83ae7..bd17815 100644 --- a/src/exlab_wizard/ui/mount.py +++ b/src/exlab_wizard/ui/mount.py @@ -26,8 +26,10 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from exlab_wizard.constants import KEYRING_USERNAME_LIMS, AuditScopeKind, RunKind +from exlab_wizard.constants import KEYRING_USERNAME_LIMS, AuditScopeKind, RunKind, RunSyncState from exlab_wizard.logging import get_logger +from exlab_wizard.orchestrator.staging_clear import clear_run_dir +from exlab_wizard.orchestrator.staging_query import list_staged_runs if TYPE_CHECKING: from fastapi import FastAPI @@ -193,7 +195,7 @@ def _on_tree_context_action(node_id: str, action: str) -> None: ui.navigate.to(f"/settings?active=equipment&equipment_id={node_id}") def _on_file_context_action(entry: Any, action: str) -> None: - _file_context_action(entry, action, ui) + _file_context_action(deps, entry, action, ui, on_done=_refresh) return main_page.render_file_explorer_page( on_open_new_project=lambda: ui.navigate.to("/wizard/project"), @@ -245,41 +247,54 @@ def _wizard_test_run() -> Any: @ui.page("/wizard/equipment") def _wizard_equipment() -> Any: - """Redesign §6 — Add-Equipment wizard route.""" + """Redesign §6 — Add-Equipment wizard route. + + ``state`` is created once for the wizard's whole lifetime: the + render layer drives Next / Back internally (re-rendering in + place), so nothing here navigates mid-wizard -- a navigation + would rebuild the page and reset every field the operator typed. + """ deps = _deps() if _restart_gate(deps, ui): return None state = wizard_equipment_page.EquipmentWizardState() - def _on_advance(current_step: str) -> None: - idx = wizard_equipment_page.EQUIPMENT_WIZARD_STEPS.index(current_step) - if idx + 1 < len(wizard_equipment_page.EQUIPMENT_WIZARD_STEPS): - state.active_step = wizard_equipment_page.EQUIPMENT_WIZARD_STEPS[idx + 1] - ui.navigate.to("/wizard/equipment") - - def _on_back(current_step: str) -> None: - idx = wizard_equipment_page.EQUIPMENT_WIZARD_STEPS.index(current_step) - if idx > 0: - state.active_step = wizard_equipment_page.EQUIPMENT_WIZARD_STEPS[idx - 1] - ui.navigate.to("/wizard/equipment") - def _on_confirm(eq: Any) -> None: - # Posts through the config router. The actual HTTP wiring is - # supplied by the deps' append-equipment callable; tests can - # stub it. - append = getattr(deps, "append_equipment", None) if deps is not None else None - if append is not None: - try: - append(eq) - except Exception as exc: - _show_toast(ui, f"Could not add equipment: {exc}", positive=False) - return + # Persist straight into the live config (Redesign §6): merge + # via the same shared helper the POST /config/equipment route + # uses, save through ``deps.save_config``, and update the + # in-memory config so the new equipment is live without a + # tray relaunch -- matching the route's no-restart contract. + from exlab_wizard.config.models import config_with_equipment_appended + from exlab_wizard.errors import ConfigError + + try: + merged = config_with_equipment_appended(getattr(deps, "config", None), eq) + except ConfigError as exc: + _show_toast(ui, f"Could not add equipment: {exc}", positive=False) + return + saver = getattr(deps, "save_config", None) if deps is not None else None + if saver is None: + _show_toast( + ui, "Cannot add equipment: no config writer is available", positive=False + ) + return + try: + result = saver(merged) + if hasattr(result, "__await__"): + # Production wires a synchronous saver; an awaitable + # here would silently no-op, so surface it. + _log.warning("save_config returned an awaitable; a sync saver is expected") + except Exception as exc: + _log.exception("append-equipment save_config failed") + _show_toast(ui, f"Could not add equipment: {exc}", positive=False) + return + if deps is not None: + deps.config = merged ui.navigate.to("/main") return wizard_equipment_page.render_wizard_equipment( state=state, - on_advance=_on_advance, - on_back=_on_back, on_confirm=_on_confirm, on_cancel=lambda: ui.navigate.to("/main"), ) @@ -322,6 +337,8 @@ def _on_create( @ui.page("/settings") def _settings(active: str = "") -> Any: + from exlab_wizard.api._dependencies import lims_password_present + deps = _deps() if _restart_gate(deps, ui): return None @@ -355,7 +372,7 @@ def _on_save(updated: Any) -> None: on_discard=None, on_save_lims_password=on_save_lims_password, on_clear_lims_password=on_clear_lims_password, - lims_password_present=bool(getattr(deps, "keyring_password_present", False)), + lims_password_present=lims_password_present(deps), ) @ui.page("/problems") @@ -427,6 +444,13 @@ def _on_save(value: str) -> None: _log.exception("LIMS keyring set_password failed") _show_toast(ui, f"Could not save the LIMS password: {exc}", positive=False) return + # The credential field re-seeds its "Set / Not set" status from + # ``deps.keyring_password_present`` on the next render, and the + # §4.9 setup gate reads the same flag. It is computed once at + # tray boot, so flip it here -- otherwise a freshly saved + # password still reads as absent until a relaunch. + if deps is not None: + deps.keyring_password_present = True _show_toast(ui, "LIMS password saved to the OS keyring", positive=True) def _on_clear() -> None: @@ -441,6 +465,10 @@ def _on_clear() -> None: _log.exception("LIMS keyring delete_password failed") _show_toast(ui, f"Could not clear the LIMS password: {exc}", positive=False) return + # Mirror of the Save path: clearing the password makes the slot + # incomplete again, so drop the boot-time flag in step. + if deps is not None: + deps.keyring_password_present = False _show_toast(ui, "LIMS password removed from the OS keyring", positive=True) return _on_save, _on_clear @@ -500,10 +528,18 @@ def _render_restart_required(ui: Any) -> Any: def _is_setup_ready(deps: Any) -> bool: - """Mirror ``api.setup.compute_setup_state`` without the API import.""" + """Mirror ``api.setup.compute_setup_state``'s readiness verdict. + + Re-evaluated here rather than calling the API so the NiceGUI mount + stays independent of the setup-state *evaluator*; the keyring read + still routes through the shared ``lims_password_present`` helper so + a change to that semantics propagates here too. + """ + from exlab_wizard.api._dependencies import lims_password_present + if deps is None or getattr(deps, "config", None) is None: return False - keyring = getattr(deps, "keyring_password_present", False) + keyring = lims_password_present(deps) lims_reachable = getattr(deps, "lims_reachable", True) return bool(keyring and lims_reachable) @@ -647,7 +683,6 @@ def _metadata_for_owned_equipment(node_id: str, config: Any) -> dict[str, Any]: "sync_mode": str(getattr(entry, "sync_mode", "")) or "nas", "local_root": entry.local_root or "", "nas_root": entry.nas_root or "", - "completeness_signal": getattr(entry, "completeness_signal", "") or "", } return {} @@ -803,6 +838,8 @@ def _drive_folder_feed(app: Any, deps: Any, selected_path: str | None) -> list[A size_bytes=entry.size_bytes, modified_iso=entry.modified_iso, sync_status=entry.sync_status, + keep_local=getattr(entry, "keep_local", False), + tombstone=getattr(entry, "tombstone", False), ) ) return entries @@ -836,12 +873,10 @@ def _run_staging_action(deps: Any, path: str, action: str, ui: Any) -> None: """Dispatch a per-run context action to its backend surface. Mirrors :func:`api.routers.staging.post_force_sync` / - :func:`api.routers.staging.post_clear` / - :func:`api.routers.browse.get_run_log` but invokes the underlying + :func:`api.routers.staging.post_clear` but invokes the underlying primitives directly from the mount so the action stays in-process (no HTTP round trip from the same Python interpreter). """ - from exlab_wizard.cache.ingest_writer import IngestWriter from exlab_wizard.ui.components.tree_context_menu import ( RUN_CONTEXT_CLEAR_VERIFIED, RUN_CONTEXT_FORCE_SYNC, @@ -871,14 +906,10 @@ async def _do_enqueue() -> None: _spawn_background(_do_enqueue()) return if action == RUN_CONTEXT_CLEAR_VERIFIED: - ingest_writer = getattr(deps, "ingest_writer", None) or IngestWriter() - from exlab_wizard.orchestrator.cleanup import clear_run async def _do_clear() -> None: try: - files, _bytes = await clear_run( - run_path, config=config, ingest_writer=ingest_writer - ) + files, _bytes = await asyncio.to_thread(clear_run_dir, run_path) except Exception as exc: _log.exception("per-run clear failed") _show_toast(ui, f"Clear failed: {exc}", positive=False) @@ -891,29 +922,37 @@ async def _do_clear() -> None: _spawn_background(_do_clear()) return if action == RUN_CONTEXT_VIEW_LOG: - _open_log_dialog(run_path, ui) + _open_log_dialog(deps, run_path, ui) return _show_toast(ui, f"Unknown staging action: {action}", positive=False) def _bulk_clear_verified(deps: Any, ui: Any) -> None: - """Run the orchestrator's bulk ``clear_all_verified`` helper. + """Bulk-clear every staged run whose sync job is verified. Wired from the file-explorer footer's *Clear verified runs* button. - Same in-process dispatch pattern as the per-run actions. + Same in-process dispatch pattern as the per-run actions. The + operator-free per-file NAS sync redesign (2026-05-21) keys the + "clearable" set off the sync-queue job state; Phase 5 swaps this to + the ``sync_state.json`` ``SYNCED`` rollup. """ - from exlab_wizard.cache.ingest_writer import IngestWriter - from exlab_wizard.orchestrator.cleanup import clear_all_verified - config = getattr(deps, "config", None) if deps is not None else None if config is None: _show_toast(ui, "Clear-verified unavailable: no config", positive=False) return - ingest_writer = getattr(deps, "ingest_writer", None) or IngestWriter() async def _do_bulk() -> None: try: - cleared = await clear_all_verified(config=config, ingest_writer=ingest_writer) + cleared: list[str] = [] + sync_state_writer = getattr(deps, "sync_state_writer", None) + for summary in list_staged_runs(config=config, sync_state_writer=sync_state_writer): + # Only a fully-SYNCED run is clearable; ``cleared`` runs + # have no staging copy left and ``syncing`` runs are unproven. + if summary.current_state != RunSyncState.SYNCED.value: + continue + files, _bytes = await asyncio.to_thread(clear_run_dir, Path(summary.path)) + if files > 0: + cleared.append(summary.path) except Exception as exc: _log.exception("bulk clear-verified failed") _show_toast(ui, f"Clear-verified failed: {exc}", positive=False) @@ -926,9 +965,20 @@ async def _do_bulk() -> None: _spawn_background(_do_bulk()) -def _file_context_action(entry: Any, action: str, ui: Any) -> None: - """Handle ``Open in OS`` / ``Copy path`` from the centre-pane file row.""" - from exlab_wizard.ui.components.file_list import FILE_CONTEXT_COPY_PATH, FILE_CONTEXT_OPEN +def _file_context_action( + deps: Any, + entry: Any, + action: str, + ui: Any, + *, + on_done: Callable[[], None] | None = None, +) -> None: + """Handle ``Open in OS`` / ``Copy path`` / ``Keep local`` file actions.""" + from exlab_wizard.ui.components.file_list import ( + FILE_CONTEXT_COPY_PATH, + FILE_CONTEXT_KEEP_LOCAL, + FILE_CONTEXT_OPEN, + ) path = str(getattr(entry, "path", "")) if not path: @@ -949,9 +999,61 @@ def _file_context_action(entry: Any, action: str, ui: Any) -> None: return _show_toast(ui, "Path copied to clipboard", positive=True) return + if action == FILE_CONTEXT_KEEP_LOCAL: + _toggle_keep_local(deps, entry, ui, on_done=on_done) + return _show_toast(ui, f"Unknown file action: {action}", positive=False) +def _toggle_keep_local( + deps: Any, + entry: Any, + ui: Any, + *, + on_done: Callable[[], None] | None = None, +) -> None: + """Flip a file's ``keep_local`` flag via the orchestrator's writer. + + Operator-free per-file NAS sync design (2026-05-21): ``sync_state.json`` + has a single writer -- the orchestrator's :class:`SyncStateWriter` -- + so the GUI never writes the file directly. The mount calls + ``set_keep_local`` in-process (matching the per-run staging actions), + which is exactly what the ``POST /staging/{run}/keep-local`` endpoint + does. The run root is resolved by walking up to the nearest + ``creation.json`` cache. + """ + from exlab_wizard.api.routers.browse import _find_run_root, _run_relative_posix + + writer = getattr(deps, "sync_state_writer", None) if deps is not None else None + if writer is None: + _show_toast(ui, "Keep-local unavailable: sync-state writer not wired", positive=False) + return + path = Path(str(getattr(entry, "path", ""))) + run_root = _find_run_root(path.parent) + if run_root is None: + _show_toast(ui, "Keep-local unavailable: file is not inside a run", positive=False) + return + rel = _run_relative_posix(run_root, path) + if rel is None: + _show_toast(ui, "Keep-local unavailable: could not resolve file path", positive=False) + return + new_value = not bool(getattr(entry, "keep_local", False)) + + async def _do_toggle() -> None: + try: + await writer.set_keep_local(run_root, rel, new_value) + except Exception as exc: + _log.exception("keep-local toggle failed") + _show_toast(ui, f"Keep-local failed: {exc}", positive=False) + return + verb = "kept local" if new_value else "no longer kept local" + _show_toast(ui, f"{path.name} {verb}", positive=True) + if on_done is not None: + on_done() + + _spawn_background(_do_toggle()) + + def _open_in_os(path: str) -> bool: """Launch the host OS's default opener for ``path``. @@ -980,46 +1082,52 @@ def _open_in_os(path: str) -> bool: return False -def _open_log_dialog(run_path: Path, ui: Any) -> None: - """Open a NiceGUI dialog showing the run's ingest.json history.""" - import msgspec +def _open_log_dialog(deps: Any, run_path: Path, ui: Any) -> None: + """Open a NiceGUI dialog showing the run's sync-queue job state. - from exlab_wizard.api.schemas import IngestJson - from exlab_wizard.io import read_msgspec_json - from exlab_wizard.paths import ingest_json_path + The operator-free per-file NAS sync redesign (2026-05-21) removed + ``ingest.json``; the per-run "log" is now the run's sync-queue job + state. Phase 5/6 source this from the ``sync_state.json`` rollup. + """ - ingest_path = ingest_json_path(run_path) - if not ingest_path.exists(): - _show_toast(ui, "No log: ingest.json not found", positive=False) - return - try: - payload = read_msgspec_json(ingest_path, IngestJson) - except (msgspec.DecodeError, msgspec.ValidationError) as exc: - _show_toast(ui, f"Log unreadable: {exc}", positive=False) - return - try: - dialog = ui.dialog() - with ( - dialog, - ui.card() - .props('data-testid="run-log-dialog"') - .style("min-width: 480px; max-width: 720px;"), - ): - ui.label(f"Log: {run_path.name}").style("font-weight: 600;") - ui.label(f"State: {payload.current_state}").style("color: var(--color-muted);") - with ui.scroll_area().style("max-height: 360px;"): - for entry in payload.history: - state_val = entry.get("state", "?") if isinstance(entry, dict) else "?" - at_val = entry.get("at", "") if isinstance(entry, dict) else "" - host_val = entry.get("host", "") if isinstance(entry, dict) else "" - ui.label(f"[{at_val}] {state_val} (host={host_val})").style( + async def _do_open() -> None: + nas_sync = getattr(deps, "nas_sync", None) if deps is not None else None + getter = getattr(nas_sync, "get_by_run_path", None) if nas_sync is not None else None + row = None + if getter is not None: + try: + row = await getter(run_path) + except Exception as exc: # pragma: no cover -- defensive + _log.warning("sync-queue lookup failed for %s: %s", run_path, exc) + state = getattr(getattr(row, "state", None), "value", None) or "none" + try: + dialog = ui.dialog() + with ( + dialog, + ui.card() + .props('data-testid="run-log-dialog"') + .style("min-width: 480px; max-width: 720px;"), + ): + ui.label(f"Log: {run_path.name}").style("font-weight: 600;") + ui.label(f"Sync state: {state}").style("color: var(--color-muted);") + if row is None: + ui.label("No sync job recorded for this run yet.").style( "font-family: var(--font-mono); font-size: 0.85em;" ) - ui.button("Close", on_click=dialog.close).props("flat") - dialog.open() - except Exception as exc: - _log.warning("log dialog render failed: %s", exc) - _show_toast(ui, "Log dialog unavailable", positive=False) + else: + for field in ("enqueued_at", "verified_at", "attempts", "last_error"): + value = getattr(row, field, None) + if value: + ui.label(f"{field}: {value}").style( + "font-family: var(--font-mono); font-size: 0.85em;" + ) + ui.button("Close", on_click=dialog.close).props("flat") + dialog.open() + except Exception as exc: + _log.warning("log dialog render failed: %s", exc) + _show_toast(ui, "Log dialog unavailable", positive=False) + + _spawn_background(_do_open()) def _missing_setup_sections(deps: Any) -> tuple[str, ...]: @@ -1029,6 +1137,8 @@ def _missing_setup_sections(deps: Any) -> tuple[str, ...]: other than READY surfaces at least one section. The Settings page uses this to auto-select the first incomplete section. """ + from exlab_wizard.api._dependencies import lims_password_present + if deps is None: return ("paths", "lims") config = getattr(deps, "config", None) @@ -1042,7 +1152,7 @@ def _missing_setup_sections(deps: Any) -> tuple[str, ...]: missing.append("paths") if not config.lims.endpoint or not config.lims.email: missing.append("lims") - if not getattr(deps, "keyring_password_present", False) and "lims" not in missing: + if not lims_password_present(deps) and "lims" not in missing: missing.append("lims") return tuple(missing) @@ -1317,9 +1427,10 @@ def _build_staging_state(deps: Any) -> Any: # Redesign §3.1: orchestrator pipeline is always active; missing # staging_root surfaces as an empty staging dock, not a None panel. try: - from exlab_wizard.orchestrator.staging_query import list_staged_runs - - rows = list_staged_runs(config=config) + rows = list_staged_runs( + config=config, + sync_state_writer=getattr(deps, "sync_state_writer", None), + ) except Exception as exc: _log.warning("staging_query failed: %s", exc) return staging_page.StagingDockState(rows=[]) diff --git a/src/exlab_wizard/ui/pages/settings.py b/src/exlab_wizard/ui/pages/settings.py index d25f14e..e188b75 100644 --- a/src/exlab_wizard/ui/pages/settings.py +++ b/src/exlab_wizard/ui/pages/settings.py @@ -14,7 +14,6 @@ from pydantic import ValidationError from exlab_wizard.config.models import Config -from exlab_wizard.constants import CompletenessSignal from exlab_wizard.logging import get_logger from exlab_wizard.ui import notifications from exlab_wizard.ui.components import credential_field, test_connection_panel @@ -438,10 +437,9 @@ def _render_equipment_section(draft: Config) -> None: ``draft.equipment`` and reflects it in the visible list; the whole draft is re-validated and persisted when the operator clicks Save. - The sub-form covers the full §9 equipment surface: a - completeness-signal radio (``sentinel_file`` / ``manifest``) that - swaps the filename field, and a transport radio (``rclone`` / - ``rsync_ssh``) that swaps the transport fieldset. + The sub-form covers the full §9 equipment surface: a transport + radio (``rclone`` / ``rsync_ssh``) that swaps the transport + fieldset. """ from nicegui import ui @@ -455,10 +453,9 @@ def _render_rows() -> None: transport_summary = ( entry.transport.type if entry.transport is not None else "stage" ) - ui.label( - f"{entry.id} -- {entry.label} " - f"[{entry.completeness_signal} / {transport_summary}]" - ).props('data-testid="settings-equipment-row"') + ui.label(f"{entry.id} -- {entry.label} [{transport_summary}]").props( + 'data-testid="settings-equipment-row"' + ) else: ui.label("No equipment configured yet.").props( 'data-testid="settings-equipment-empty"' @@ -473,30 +470,9 @@ def _render_rows() -> None: eq_local = ui.input(label="Local root").props('data-testid="settings-equipment-local-root"') eq_nas = ui.input(label="NAS root").props('data-testid="settings-equipment-nas-root"') - # Completeness signal: a radio that swaps the filename field. - signal_radio = ui.radio( - [CompletenessSignal.SENTINEL_FILE.value, CompletenessSignal.MANIFEST.value], - value=CompletenessSignal.SENTINEL_FILE.value, - ).props('data-testid="settings-equipment-signal"') # Widget refs the swap-panels and ``_add`` share. fields: dict[str, Any] = {} - @ui.refreshable - def _signal_field() -> None: - if signal_radio.value == CompletenessSignal.MANIFEST.value: - fields["manifest"] = ui.input(label="Manifest filename", value="manifest.json").props( - 'data-testid="settings-equipment-manifest"' - ) - fields.pop("sentinel", None) - else: - fields["sentinel"] = ui.input( - label="Sentinel filename", value="acquisition_complete.flag" - ).props('data-testid="settings-equipment-sentinel"') - fields.pop("manifest", None) - - _signal_field() - signal_radio.on_value_change(lambda _e: _signal_field.refresh()) - # Transport: a radio that swaps the transport fieldset. transport_radio = ui.radio(["rclone", "rsync_ssh"], value="rclone").props( 'data-testid="settings-equipment-transport"' @@ -536,9 +512,6 @@ def _add(_evt: Any = None) -> None: label=eq_label.value or "", local_root=eq_local.value or "", nas_root=eq_nas.value or "", - completeness_signal=signal_radio.value or CompletenessSignal.SENTINEL_FILE.value, - sentinel_filename=(fields["sentinel"].value or "" if "sentinel" in fields else ""), - manifest_filename=(fields["manifest"].value or "" if "manifest" in fields else ""), transport_type=transport_radio.value or "rclone", rclone_remote=( fields["rclone_remote"].value or "" if "rclone_remote" in fields else "" diff --git a/src/exlab_wizard/ui/pages/staging.py b/src/exlab_wizard/ui/pages/staging.py index 5e140ae..a201376 100644 --- a/src/exlab_wizard/ui/pages/staging.py +++ b/src/exlab_wizard/ui/pages/staging.py @@ -29,7 +29,7 @@ from dataclasses import dataclass from typing import Any -from exlab_wizard.constants import IngestState +from exlab_wizard.constants import RunSyncState from exlab_wizard.logging import get_logger from exlab_wizard.orchestrator.staging_query import StagedRunSummary @@ -64,17 +64,23 @@ """The seven columns displayed (column order is part of the spec).""" -# State -> color mapping mirroring the design tokens in +# Run rollup state -> color mapping mirroring the design tokens in # ``exlab_wizard.ui.design``. Kept here as plain strings so the unit tests -# don't depend on the full design module being importable. +# don't depend on the full design module being importable. Phase 5 of the +# operator-free per-file NAS sync redesign (2026-05-21) sources +# ``current_state`` from the derived ``sync_state.json`` +# ``RunSyncState`` rollup -- ``syncing`` / ``synced`` / ``cleared``. _STATE_COLORS: dict[str, str] = { - IngestState.STAGING.value: "var(--color-info)", - IngestState.COMPLETE.value: "var(--color-success)", - IngestState.SYNC_QUEUED.value: "var(--color-info)", - IngestState.SYNC_VERIFIED.value: "var(--color-success)", - IngestState.CLEARED.value: "var(--color-muted)", + RunSyncState.SYNCING.value: "var(--color-info)", + RunSyncState.SYNCED.value: "var(--color-success)", + RunSyncState.CLEARED.value: "var(--color-muted)", } +# Run rollup states whose staging copy may still be cleared: only a +# fully-``synced`` run (``cleared`` has no staging copy left, ``syncing`` +# is unproven). +_CLEARABLE_STATES: frozenset[str] = frozenset({RunSyncState.SYNCED.value}) + @dataclass class StagingDockState: @@ -164,7 +170,7 @@ def row_props(row: StagedRunSummary) -> dict[str, Any]: "files": row.file_count, "bytes": format_bytes(row.byte_total), "elapsed": format_elapsed(row.elapsed_seconds_since_last_activity), - "is_clearable": row.current_state == IngestState.SYNC_VERIFIED.value, + "is_clearable": row.current_state in _CLEARABLE_STATES, } @@ -216,9 +222,7 @@ def render_staging_dock(state: StagingDockState) -> Any: "font-weight: 600;", ) ui.space() - verified_count = sum( - 1 for row in state.rows if row.current_state == IngestState.SYNC_VERIFIED.value - ) + verified_count = sum(1 for row in state.rows if row.current_state in _CLEARABLE_STATES) ui.button( f"Clear verified runs ({verified_count})", on_click=lambda _evt: _invoke(state.on_clear_verified), diff --git a/src/exlab_wizard/ui/pages/wizard_equipment.py b/src/exlab_wizard/ui/pages/wizard_equipment.py index 25840f0..8ee8b9f 100644 --- a/src/exlab_wizard/ui/pages/wizard_equipment.py +++ b/src/exlab_wizard/ui/pages/wizard_equipment.py @@ -1,6 +1,6 @@ """Add-Equipment wizard (GUI/Orchestrator Redesign §6). -Five-step wizard launched from the main-window toolbar: +Four-step wizard launched from the main-window toolbar: 1. Identity — equipment ID (validated against ``^[A-Z][A-Z0-9_]*$``) + label. @@ -8,8 +8,7 @@ 3. Sync mode — pick ``nas`` (acquire + sync directly to NAS) or ``stage`` (acquire + push to a connected PC's staging area). The step then shows the matching transport sub-form. -4. Completeness signal — sentinel vs manifest + filename. -5. Review & confirm — assembles a validated EquipmentConfig via the +4. Review & confirm — assembles a validated EquipmentConfig via the shared ``build_equipment_config()`` and posts it through ``POST /config/equipment``. @@ -35,7 +34,6 @@ "identity", "paths", "sync_mode", - "signal", "review", ) @@ -43,7 +41,6 @@ "identity": "Identity", "paths": "Paths", "sync_mode": "Sync mode", - "signal": "Completeness signal", "review": "Review & confirm", } @@ -71,10 +68,6 @@ class EquipmentWizardState: staging_mount_point: str = "" staging_subpath: str = "" # Step 4 - completeness_signal: str = "sentinel_file" - sentinel_filename: str = "" - manifest_filename: str = "" - # Step 5 last_error: str | None = None confirmed: bool = False @@ -101,10 +94,6 @@ def can_advance(state: EquipmentWizardState) -> bool: return bool(state.ssh_target.strip() and state.rsync_remote_path.strip()) # stage return bool(state.staging_mount_point.strip() and state.staging_subpath.strip()) - case "signal": - if state.completeness_signal == "sentinel_file": - return bool(state.sentinel_filename.strip()) - return bool(state.manifest_filename.strip()) case "review": return True return False @@ -123,9 +112,6 @@ def assemble_equipment_config( label=state.label, local_root=state.local_root, nas_root=state.nas_root, - completeness_signal=state.completeness_signal, - sentinel_filename=state.sentinel_filename, - manifest_filename=state.manifest_filename, sync_mode=state.sync_mode, transport_type=state.transport_type, rclone_remote=state.rclone_remote, @@ -142,17 +128,22 @@ def assemble_equipment_config( def render_wizard_equipment( *, state: EquipmentWizardState | None = None, - on_advance: Callable[[str], None] | None = None, - on_back: Callable[[str], None] | None = None, on_confirm: Callable[[EquipmentConfig], None] | None = None, on_cancel: Callable[[], None] | None = None, ) -> Any: # pragma: no cover -- NiceGUI render, driven by e2e - """Render the Add-Equipment wizard. Pure render function. - - Rendered as a full-page card (not a dialog) since the wizard has its - own route at ``/wizard/equipment``; the previous dialog wrapping - required an explicit ``.open()`` call which the route handler had - no clean place to issue. + """Render the self-contained Add-Equipment wizard. + + Next / Back navigation is handled *inside* the render: the step body + and footer live in one ``@ui.refreshable`` so advancing mutates + ``state.active_step`` and re-renders in place. The wizard therefore + keeps a single ``EquipmentWizardState`` for its whole lifetime -- + the caller creates it once and never round-trips through a page + navigation that would reset it. Only ``on_confirm`` (post the + assembled ``EquipmentConfig``) and ``on_cancel`` (leave the wizard) + cross back to the host. + + Rendered as a full-page card since the wizard owns the + ``/wizard/equipment`` route. """ s = state or EquipmentWizardState() @@ -161,44 +152,80 @@ def render_wizard_equipment( except Exception: return {"state": s} + # Handle to the live Next button. ``_body`` rewrites this on every + # re-render; ``_sync_next`` toggles the button's enabled state as the + # operator edits a step -- without a re-render, so input focus is + # kept while typing. + next_btn: dict[str, Any] = {} + with ui.card().classes("w-full h-full p-6").props('data-testid="wizard-equipment"') as dialog: ui.label("Add Equipment").style( "font-family: var(--font-display); font-size: var(--text-lg); " "color: var(--color-heading); font-weight: 600;" ) - ui.label(EQUIPMENT_STEP_TITLES[s.active_step]).style( - "color: var(--color-muted); margin-bottom: var(--sp-3);" - ).props(f'data-testid="wizard-equipment-step-{s.active_step}"') - - _STEP_RENDERERS[s.active_step](s) - - with ( - ui.row() - .classes("items-center w-full") - .style("margin-top: var(--sp-4); gap: var(--sp-2);") - ): - if on_cancel is not None: - ui.button("Cancel").props('flat data-testid="wizard-equipment-cancel"').on( - "click", lambda _evt: on_cancel() - ) - if s.active_step != EQUIPMENT_WIZARD_STEPS[0] and on_back is not None: - ui.button("Back").props('flat data-testid="wizard-equipment-back"').on( - "click", lambda _evt: on_back(s.active_step) - ) - ui.space() - if s.active_step == "review": - ui.button("Confirm").props( - 'color=primary data-testid="wizard-equipment-confirm"' - ).on( - "click", - lambda _evt: _maybe_confirm(s, on_confirm), - ) - else: - btn = ui.button("Next").props('color=primary data-testid="wizard-equipment-next"') - if on_advance is not None: - btn.on("click", lambda _evt: on_advance(s.active_step)) - if not can_advance(s): - btn.props("disable") + + def _sync_next() -> None: + """Re-evaluate ``can_advance`` and enable/disable Next in place.""" + btn = next_btn.get("btn") + if btn is not None: + btn.set_enabled(can_advance(s)) + + def _step_forward() -> None: + """Advance one step, but only when the current step is valid.""" + if not can_advance(s): + return + idx = EQUIPMENT_WIZARD_STEPS.index(s.active_step) + if idx + 1 < len(EQUIPMENT_WIZARD_STEPS): + s.active_step = EQUIPMENT_WIZARD_STEPS[idx + 1] + _body.refresh() + + def _step_back() -> None: + """Return to the previous step, keeping every entered value.""" + idx = EQUIPMENT_WIZARD_STEPS.index(s.active_step) + if idx > 0: + s.active_step = EQUIPMENT_WIZARD_STEPS[idx - 1] + _body.refresh() + + @ui.refreshable + def _body() -> None: + next_btn.pop("btn", None) + ui.label(EQUIPMENT_STEP_TITLES[s.active_step]).style( + "color: var(--color-muted); margin-bottom: var(--sp-3);" + ).props(f'data-testid="wizard-equipment-step-{s.active_step}"') + + # Step renderers wire radios to ``_body.refresh`` (a changed + # radio swaps which sub-form is shown) and text inputs to + # ``_sync_next`` (re-checks the Next gate without a re-render). + _STEP_RENDERERS[s.active_step](s, _body.refresh, _sync_next) + + with ( + ui.row() + .classes("items-center w-full") + .style("margin-top: var(--sp-4); gap: var(--sp-2);") + ): + if on_cancel is not None: + cancel_cb = on_cancel + ui.button("Cancel").props('flat data-testid="wizard-equipment-cancel"').on( + "click", lambda _evt: cancel_cb() + ) + if s.active_step != EQUIPMENT_WIZARD_STEPS[0]: + ui.button("Back").props('flat data-testid="wizard-equipment-back"').on( + "click", lambda _evt: _step_back() + ) + ui.space() + if s.active_step == "review": + ui.button("Confirm").props( + 'color=primary data-testid="wizard-equipment-confirm"' + ).on("click", lambda _evt: _maybe_confirm(s, on_confirm)) + else: + btn = ui.button("Next").props( + 'color=primary data-testid="wizard-equipment-next"' + ) + btn.on("click", lambda _evt: _step_forward()) + btn.set_enabled(can_advance(s)) + next_btn["btn"] = btn + + _body() return dialog @@ -219,99 +246,98 @@ def _maybe_confirm( def _render_identity_step( state: EquipmentWizardState, + refresh_body: Callable[[], object], + sync_next: Callable[[], object], ) -> None: # pragma: no cover -- NiceGUI render, driven by e2e + del refresh_body # identity has no structural (radio) controls try: from nicegui import ui except Exception: return - ui.input(label="Equipment ID (^[A-Z][A-Z0-9_]*$)").props( + ui.input(label="Equipment ID (^[A-Z][A-Z0-9_]*$)", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-id"' ).bind_value(state, "equipment_id") - ui.input(label="Label").props('data-testid="wizard-equipment-label"').bind_value(state, "label") + ui.input(label="Label", on_change=lambda _e: sync_next()).props( + 'data-testid="wizard-equipment-label"' + ).bind_value(state, "label") def _render_paths_step( state: EquipmentWizardState, + refresh_body: Callable[[], object], + sync_next: Callable[[], object], ) -> None: # pragma: no cover -- NiceGUI render, driven by e2e + del refresh_body # paths has no structural (radio) controls try: from nicegui import ui except Exception: return - ui.input(label="Local root").props('data-testid="wizard-equipment-local-root"').bind_value( - state, "local_root" - ) - ui.input(label="NAS root").props('data-testid="wizard-equipment-nas-root"').bind_value( - state, "nas_root" - ) + ui.input(label="Local root", on_change=lambda _e: sync_next()).props( + 'data-testid="wizard-equipment-local-root"' + ).bind_value(state, "local_root") + ui.input(label="NAS root", on_change=lambda _e: sync_next()).props( + 'data-testid="wizard-equipment-nas-root"' + ).bind_value(state, "nas_root") def _render_sync_mode_step( state: EquipmentWizardState, + refresh_body: Callable[[], object], + sync_next: Callable[[], object], ) -> None: # pragma: no cover -- NiceGUI render, driven by e2e try: from nicegui import ui except Exception: return with ui.row().classes("items-center"): - ui.radio(["nas", "stage"], value=state.sync_mode).props( - 'data-testid="wizard-equipment-sync-mode"' - ).bind_value(state, "sync_mode") + ui.radio( + ["nas", "stage"], value=state.sync_mode, on_change=lambda _e: refresh_body() + ).props('data-testid="wizard-equipment-sync-mode"').bind_value(state, "sync_mode") if state.sync_mode == "nas": - ui.radio(["rclone", "rsync_ssh"], value=state.transport_type).props( - 'data-testid="wizard-equipment-transport-type"' - ).bind_value(state, "transport_type") + ui.radio( + ["rclone", "rsync_ssh"], + value=state.transport_type, + on_change=lambda _e: refresh_body(), + ).props('data-testid="wizard-equipment-transport-type"').bind_value(state, "transport_type") if state.transport_type == "rclone": - ui.input(label="rclone remote").props( + ui.input(label="rclone remote", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-rclone-remote"' ).bind_value(state, "rclone_remote") - ui.input(label="rclone remote path").props( + ui.input(label="rclone remote path", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-rclone-remote-path"' ).bind_value(state, "rclone_remote_path") else: - ui.input(label="SSH target").props( + ui.input(label="SSH target", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-ssh-target"' ).bind_value(state, "ssh_target") - ui.input(label="SSH key path").props( + ui.input(label="SSH key path", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-ssh-key-path"' ).bind_value(state, "ssh_key_path") - ui.input(label="rsync remote path").props( + ui.input(label="rsync remote path", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-rsync-remote-path"' ).bind_value(state, "rsync_remote_path") else: # stage - ui.radio(["smb_mount", "file_transfer"], value=state.staging_transport_type).props( - 'data-testid="wizard-equipment-staging-transport-type"' - ).bind_value(state, "staging_transport_type") - ui.input(label="Mount point").props( + ui.radio( + ["smb_mount", "file_transfer"], + value=state.staging_transport_type, + on_change=lambda _e: refresh_body(), + ).props('data-testid="wizard-equipment-staging-transport-type"').bind_value( + state, "staging_transport_type" + ) + ui.input(label="Mount point", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-staging-mount-point"' ).bind_value(state, "staging_mount_point") - ui.input(label="Staging subpath").props( + ui.input(label="Staging subpath", on_change=lambda _e: sync_next()).props( 'data-testid="wizard-equipment-staging-subpath"' ).bind_value(state, "staging_subpath") -def _render_signal_step( - state: EquipmentWizardState, -) -> None: # pragma: no cover -- NiceGUI render, driven by e2e - try: - from nicegui import ui - except Exception: - return - ui.radio(["sentinel_file", "manifest"], value=state.completeness_signal).props( - 'data-testid="wizard-equipment-signal"' - ).bind_value(state, "completeness_signal") - if state.completeness_signal == "sentinel_file": - ui.input(label="Sentinel filename").props( - 'data-testid="wizard-equipment-sentinel-filename"' - ).bind_value(state, "sentinel_filename") - else: - ui.input(label="Manifest filename").props( - 'data-testid="wizard-equipment-manifest-filename"' - ).bind_value(state, "manifest_filename") - - def _render_review_step( state: EquipmentWizardState, + refresh_body: Callable[[], object], + sync_next: Callable[[], object], ) -> None: # pragma: no cover -- NiceGUI render, driven by e2e + del refresh_body, sync_next # review is a static summary, no inputs try: from nicegui import ui except Exception: @@ -329,20 +355,17 @@ def _render_review_step( ui.label(f"Staging transport: {state.staging_transport_type}") ui.label(f"Mount point: {state.staging_mount_point}") ui.label(f"Staging subpath: {state.staging_subpath}") - ui.label( - f"Completeness signal: {state.completeness_signal} / " - + (state.sentinel_filename or state.manifest_filename) - ) if state.last_error: ui.label(f"Error: {state.last_error}").style("color: var(--color-danger);").props( 'data-testid="wizard-equipment-error"' ) -_STEP_RENDERERS: dict[str, Callable[[EquipmentWizardState], None]] = { +_StepRenderer = Callable[[EquipmentWizardState, Callable[[], object], Callable[[], object]], None] + +_STEP_RENDERERS: dict[str, _StepRenderer] = { "identity": _render_identity_step, "paths": _render_paths_step, "sync_mode": _render_sync_mode_step, - "signal": _render_signal_step, "review": _render_review_step, } diff --git a/tests/e2e/_test_app.py b/tests/e2e/_test_app.py index 3d1afd4..f28b6c8 100644 --- a/tests/e2e/_test_app.py +++ b/tests/e2e/_test_app.py @@ -91,7 +91,10 @@ class TestState: selected_node_is_received: bool = False # Seeded folder-feed payload keyed by tree-node id; the test app # serves these as the centre-pane file list. - folder_feeds: dict[str, list[tuple[str, int, str | None]]] = field(default_factory=dict) + # Each row is a 3-tuple ``(name, size, sync)`` or a 5-tuple that + # additionally carries ``(keep_local, tombstone)`` -- the operator-free + # per-file NAS sync design (2026-05-21) added the latter two fields. + folder_feeds: dict[str, list[tuple[Any, ...]]] = field(default_factory=dict) # Seeded findings that the travelling-badge flow consumes (path → tier). seeded_findings: list[tuple[str, str]] = field(default_factory=list) @@ -134,7 +137,6 @@ def _seeded_metadata_payload(node_id: str | None, node_kind: str | None) -> dict "sync_mode": "stage" if "stage" in node_id else "nas", "local_root": "/data/lab", "nas_root": "//nas/lab", - "completeness_signal": "sentinel_file", } if node_kind == "received_equipment": return { @@ -168,30 +170,55 @@ def _seeded_metadata_payload(node_id: str | None, node_kind: str | None) -> dict return {} +# Default synthetic file-list feed. Operator-free per-file NAS sync +# design (2026-05-21): the rows exercise the five per-file display +# states -- a synced+kept-local file, an acquiring file, a syncing +# file, and an "On NAS" tombstone (cleared run, no local copy). Each +# tuple is (name, size|None, sync_status, keep_local, tombstone). +_DEFAULT_FEED_ROWS: list[tuple[str, int | None, str | None, bool, bool]] = [ + ("scan.tif", 1024, "synced", True, False), + ("metadata.json", 256, "acquiring", False, False), + ("frames.raw", 4096, "syncing", False, False), + ("archived.tif", None, "on_nas", False, True), +] + + def _seeded_file_entries(test_state: TestState, node_id: str | None) -> list[Any]: """Build the centre-pane file rows the test flows assert on. - Returns a default synthetic two-file feed unless the test seeded a - specific path via ``test_state.folder_feeds``. + Returns a default synthetic feed (covering every per-file display + state, including a keep-local file and an "On NAS" tombstone) unless + the test seeded a specific path via ``test_state.folder_feeds``. A + seeded 3-tuple ``(name, size, sync)`` keeps backward compatibility; a + 5-tuple additionally carries ``keep_local`` / ``tombstone``. """ if node_id is None: return [] from exlab_wizard.ui.components.file_list import FileListEntry - rows = test_state.folder_feeds.get( - node_id, - [("scan.tif", 1024, "synced"), ("metadata.json", 256, "pending")], - ) + seeded = test_state.folder_feeds.get(node_id) + rows: list[tuple[str, int | None, str | None, bool, bool]] + if seeded is None: + rows = list(_DEFAULT_FEED_ROWS) + else: + rows = [ + (row[0], row[1], row[2], False, False) + if len(row) == 3 + else (row[0], row[1], row[2], row[3], row[4]) + for row in seeded + ] return [ FileListEntry( name=name, path=f"{node_id}/{name}", is_dir=False, size_bytes=size, - modified_iso="2026-05-14T09:22:00Z", + modified_iso=None if tombstone else "2026-05-14T09:22:00Z", sync_status=sync, + keep_local=keep_local, + tombstone=tombstone, ) - for (name, size, sync) in rows + for (name, size, sync, keep_local, tombstone) in rows ] @@ -285,8 +312,11 @@ def main_index( tree_component.RunNode( directory_name="Run_2026-05-06", run_kind="experimental", - label="Cleaned run", - sync_status="cleaned", + label="Cleared run", + # Operator-free per-file NAS sync design + # (2026-05-21): the run rollup is a RunSyncState + # value; ``cleared`` drives the cloud icon. + sync_status="cleared", ), tree_component.RunNode("TestRun_2026-05-07", "test", "Test run"), ], @@ -468,41 +498,47 @@ def _submit(state: wizard_run_page.RunWizardState) -> None: # Add-Equipment wizard (Flow 16 -- Redesign §6) # ---------------------------------------------------------------------- @ui.page("/wizard/equipment") - def wizard_equipment_index(step: str = "identity") -> None: - state = wizard_equipment_page.EquipmentWizardState( - active_step=step or "identity", - equipment_id="FLOW_99", - label="Flow Cytometer 99", - local_root="/data", - nas_root="/srv/nas", - rclone_remote="lab-nas", - rclone_remote_path="lab/FLOW_99", - sentinel_filename="done.flag", - ) - - def _advance(current: str) -> None: - idx = wizard_equipment_page.EQUIPMENT_WIZARD_STEPS.index(current) - if idx + 1 < len(wizard_equipment_page.EQUIPMENT_WIZARD_STEPS): - ui.navigate.to( - f"/wizard/equipment?step={wizard_equipment_page.EQUIPMENT_WIZARD_STEPS[idx + 1]}" - ) - - def _back(current: str) -> None: - idx = wizard_equipment_page.EQUIPMENT_WIZARD_STEPS.index(current) - if idx > 0: - ui.navigate.to( - f"/wizard/equipment?step={wizard_equipment_page.EQUIPMENT_WIZARD_STEPS[idx - 1]}" - ) + def wizard_equipment_index(step: str = "identity", seed: str = "1") -> None: + # The render layer now drives Next / Back internally, so this + # route wires the wizard exactly as production does -- one state, + # ``on_confirm`` / ``on_cancel`` only. ``seed=1`` (default) + # pre-fills a valid state so a step can be deep-linked and its + # render asserted; ``seed=0`` starts empty to exercise the real + # type -> Next -> advance path. + if seed == "0": + state = wizard_equipment_page.EquipmentWizardState(active_step=step or "identity") + else: + state = wizard_equipment_page.EquipmentWizardState( + active_step=step or "identity", + equipment_id="FLOW_99", + label="Flow Cytometer 99", + local_root="/data", + nas_root="/srv/nas", + rclone_remote="lab-nas", + rclone_remote_path="lab/FLOW_99", + ) def _confirm(eq: Any) -> None: + # Mirror the production confirm wiring (mount.py): merge via + # the shared helper and persist, rather than just stashing + # the raw equipment -- so this surface no longer masks the + # real append path. + from exlab_wizard.config.models import config_with_equipment_appended + from exlab_wizard.errors import ConfigError + + try: + merged = config_with_equipment_appended(test_state.config, eq) + except ConfigError as exc: + ui.label(f"Error: {exc}").props('data-testid="wizard-equipment-error"') + return + test_state.config = merged + test_state.saved_config = merged test_state.appended_equipment = eq test_state.last_action = "wizard.equipment.confirm" ui.label("Equipment added").props('data-testid="wizard-equipment-success"') wizard_equipment_page.render_wizard_equipment( state=state, - on_advance=_advance, - on_back=_back, on_confirm=_confirm, on_cancel=lambda: ui.navigate.to("/main"), ) @@ -671,9 +707,9 @@ def _revoke(finding_id: str) -> None: # Staging dock (Flow 09) # ---------------------------------------------------------------------- @ui.page("/staging") - def staging_index(state: str = "staging") -> None: - from exlab_wizard.constants import IngestState + def staging_index(state: str = "none") -> None: from exlab_wizard.orchestrator.staging_query import StagedRunSummary + from exlab_wizard.sync.queue import SyncJobState rows = [ StagedRunSummary( @@ -691,11 +727,11 @@ def staging_index(state: str = "staging") -> None: def _force_sync(path: str) -> None: test_state.last_action = f"staging.force_sync:{path}" - ui.navigate.to(f"/staging?state={IngestState.SYNC_QUEUED.value}") + ui.navigate.to(f"/staging?state={SyncJobState.QUEUED.value}") def _clear(path: str) -> None: test_state.last_action = f"staging.clear:{path}" - ui.navigate.to(f"/staging?state={IngestState.CLEARED.value}") + ui.navigate.to(f"/staging?state={SyncJobState.CLEANED.value}") def _view_log(path: str) -> None: test_state.last_action = f"staging.view_log:{path}" diff --git a/tests/e2e/page_objects/wizard_equipment_page.py b/tests/e2e/page_objects/wizard_equipment_page.py index 4675a51..74c814d 100644 --- a/tests/e2e/page_objects/wizard_equipment_page.py +++ b/tests/e2e/page_objects/wizard_equipment_page.py @@ -22,6 +22,10 @@ def equipment_id(self) -> Any: def label(self) -> Any: return self._page.locator('[data-testid="wizard-equipment-label"]') + @property + def step_paths(self) -> Any: + return self._page.locator('[data-testid="wizard-equipment-step-paths"]') + # Paths @property def local_root(self) -> Any: @@ -36,14 +40,13 @@ def nas_root(self) -> Any: def sync_mode(self) -> Any: return self._page.locator('[data-testid="wizard-equipment-sync-mode"]') - # Signal @property - def signal(self) -> Any: - return self._page.locator('[data-testid="wizard-equipment-signal"]') + def rclone_remote(self) -> Any: + return self._page.locator('[data-testid="wizard-equipment-rclone-remote"]') @property - def sentinel_filename(self) -> Any: - return self._page.locator('[data-testid="wizard-equipment-sentinel-filename"]') + def rclone_remote_path(self) -> Any: + return self._page.locator('[data-testid="wizard-equipment-rclone-remote-path"]') # Review / confirm @property @@ -58,6 +61,10 @@ def cancel(self) -> Any: def next_button(self) -> Any: return self._page.locator('[data-testid="wizard-equipment-next"]') + @property + def back(self) -> Any: + return self._page.locator('[data-testid="wizard-equipment-back"]') + @property def success(self) -> Any: return self._page.locator('[data-testid="wizard-equipment-success"]') diff --git a/tests/e2e/test_flow_00_fresh_install_setup.py b/tests/e2e/test_flow_00_fresh_install_setup.py index fcea5c3..fe65395 100644 --- a/tests/e2e/test_flow_00_fresh_install_setup.py +++ b/tests/e2e/test_flow_00_fresh_install_setup.py @@ -16,11 +16,11 @@ -> /restart-required gate -> config.yaml written under the tmp HOME with the entered values -The equipment / project / run / test-run / template creation flows are -NOT covered here: their wizard submit handlers in -``exlab_wizard/ui/mount.py`` are still toast-only stubs and no template -UX exists yet, so there is nothing functional to drive end-to-end. -Wiring those is tracked as follow-up feature work. +The project / run / test-run / template creation flows are NOT covered +here: their wizard submit handlers in ``exlab_wizard/ui/mount.py`` are +still toast-only stubs and no template UX exists yet. The Add-Equipment +wizard's confirm now persists for real -- that is exercised against the +production app in ``test_flow_26_equipment_wizard_persist.py``. """ from __future__ import annotations diff --git a/tests/e2e/test_flow_00_full_lifecycle.py b/tests/e2e/test_flow_00_full_lifecycle.py index 8fc9208..89638e4 100644 --- a/tests/e2e/test_flow_00_full_lifecycle.py +++ b/tests/e2e/test_flow_00_full_lifecycle.py @@ -220,26 +220,23 @@ def test_full_create_lifecycle(browser, prod_server: ProdServer, tmp_path: Path) _fill(page, "settings-lims-offline-path", str(catalogue_path)) # ---- Phase 4: add equipment ------------------------------------ - # 4a. rclone transport + sentinel_file signal (the default radios). + # 4a. rclone transport (the default radio). page.get_by_test_id("settings-nav-equipment").click() _fill(page, "settings-equipment-id", "MICROSCOPE1") _fill(page, "settings-equipment-label", "Confocal Microscope 1") _fill(page, "settings-equipment-local-root", str(data_root)) _fill(page, "settings-equipment-nas-root", "/srv/nas/microscope1") - _fill(page, "settings-equipment-sentinel", "acquisition_complete.flag") _fill(page, "settings-equipment-rclone-remote", "lab-nas") _fill(page, "settings-equipment-rclone-path", "lab/microscope1") page.get_by_test_id("settings-equipment-add").click() page.get_by_test_id("settings-equipment-row").first.wait_for(state="visible", timeout=8_000) - # 4b. rsync_ssh transport + manifest signal -- exercises the - # completeness-signal and transport radios swapping fields. + # 4b. rsync_ssh transport -- exercises the transport radio + # swapping the transport fieldset. _fill(page, "settings-equipment-id", "SPECTROMETER1") _fill(page, "settings-equipment-label", "Mass Spectrometer 1") _fill(page, "settings-equipment-local-root", str(data_root)) _fill(page, "settings-equipment-nas-root", "/srv/nas/spectrometer1") - _pick_radio(page, "settings-equipment-signal", "manifest") - _fill(page, "settings-equipment-manifest", "manifest.json") _pick_radio(page, "settings-equipment-transport", "rsync_ssh") _fill(page, "settings-equipment-ssh-target", "operator@nas.example.test") _fill(page, "settings-equipment-rsync-path", "/srv/nas/spectrometer1/incoming") @@ -258,7 +255,6 @@ def test_full_create_lifecycle(browser, prod_server: ProdServer, tmp_path: Path) assert "MICROSCOPE1" in config_text assert "SPECTROMETER1" in config_text assert "rsync_ssh" in config_text - assert "manifest.json" in config_text assert str(data_root) in config_text # ---- Phase 6: restart so the controller picks up the config ---- diff --git a/tests/e2e/test_flow_05_browse_view_sync_icons.py b/tests/e2e/test_flow_05_browse_view_sync_icons.py index 84b3088..45d407b 100644 --- a/tests/e2e/test_flow_05_browse_view_sync_icons.py +++ b/tests/e2e/test_flow_05_browse_view_sync_icons.py @@ -3,8 +3,8 @@ Verifies that the project / equipment tree (Frontend §3.5) renders the correct SVG icon to the left of each run name based on its sync status: -* ``sync_status != "cleaned"`` (or absent) -> ``/assets/sync_local.svg`` -* ``sync_status == "cleaned"`` -> ``/assets/sync_cloud.svg`` +* rollup other than ``cleared`` (or absent) -> ``/assets/sync_local.svg`` +* rollup ``cleared`` -> ``/assets/sync_cloud.svg`` Also asserts the static asset mount actually serves the SVGs (200 OK) so a missing PyInstaller bundle entry would surface here. @@ -27,7 +27,7 @@ def test_flow_05_sync_icons_render_in_tree(page, server_url) -> None: # The seeded hierarchy in tests/e2e/_test_app.py contains: # - Run_2026-05-07 (local; sync_status=None) -> sync_local.svg - # - Run_2026-05-06 (cleaned; sync_status=cleaned) -> sync_cloud.svg + # - Run_2026-05-06 (cleared; sync_status=cleared) -> sync_cloud.svg # - TestRun_2026-05-07 (local; sync_status=None) -> sync_local.svg local_icons = tree.locator('img[src="/assets/sync_local.svg"]') cloud_icons = tree.locator('img[src="/assets/sync_cloud.svg"]') @@ -39,8 +39,8 @@ def test_flow_05_sync_icons_render_in_tree(page, server_url) -> None: # The cleaned-run icon's parent header carries the canonical sync_status # marker on the label span (set by the default-header slot template). cloud_header = cloud_icons.first.locator("xpath=..") - assert cloud_header.locator('span[data-sync-status="cleaned"]').count() == 1, ( - "cleaned run header missing data-sync-status='cleaned' marker" + assert cloud_header.locator('span[data-sync-status="cleared"]').count() == 1, ( + "cleared run header missing data-sync-status='cleared' marker" ) diff --git a/tests/e2e/test_flow_09_orchestrator.py b/tests/e2e/test_flow_09_orchestrator.py index f95acbd..6a10723 100644 --- a/tests/e2e/test_flow_09_orchestrator.py +++ b/tests/e2e/test_flow_09_orchestrator.py @@ -1,14 +1,15 @@ -"""E2E flow 09: Orchestrator staging panel + ingest.json watcher. +"""E2E flow 09: Orchestrator staging panel + quiescence-sync trigger. -Frontend Spec §6.8 (staging panel), Backend Spec §13 (orchestrator -mode + ingest.json contract). +Frontend Spec §6.8 (staging panel), Backend Spec §13 (orchestrator mode). +The operator-free per-file NAS sync redesign (2026-05-21) replaced the +``ingest.json`` five-state machine with the sync-queue job state. The flow: -1. Plant a staged run via the test app's ``/staging?state=staging`` route. -2. Verify the staging row renders with the right state pill. +1. Plant a staged run via the test app's ``/staging?state=none`` route. +2. Verify the staging row renders with its state pill. 3. Click ``Force sync`` and verify the row's state advances to - ``sync_queued``. + ``queued``. """ from __future__ import annotations @@ -18,14 +19,14 @@ def test_flow_09_orchestrator(page, server_url) -> None: staging = StagingPage(page) - page.goto(f"{server_url}/staging?state=staging") + page.goto(f"{server_url}/staging?state=none") page.wait_for_load_state("networkidle") staging.dock.wait_for(state="visible", timeout=10_000) staging.row(0).wait_for(state="visible", timeout=5_000) - assert "staging" in staging.row(0).inner_text().lower() + assert "none" in staging.row(0).inner_text().lower() staging.force_sync(0).click() page.wait_for_load_state("networkidle") staging.row(0).wait_for(state="visible", timeout=5_000) - assert "sync_queued" in staging.row(0).inner_text().lower() + assert "queued" in staging.row(0).inner_text().lower() diff --git a/tests/e2e/test_flow_16_add_equipment.py b/tests/e2e/test_flow_16_add_equipment.py index d4ca10c..905f6b9 100644 --- a/tests/e2e/test_flow_16_add_equipment.py +++ b/tests/e2e/test_flow_16_add_equipment.py @@ -1,7 +1,7 @@ """E2E flow 16: Add-Equipment wizard (Redesign §6). -Drives the five-step wizard end to end against the test app: -identity → paths → sync_mode → signal → review → confirm. +Drives the four-step wizard end to end against the test app: +identity → paths → sync_mode → review → confirm. The test app mounts the wizard at ``/wizard/equipment?step=`` so each step can be loaded directly; the production navigation between @@ -12,6 +12,8 @@ from __future__ import annotations +from playwright.sync_api import expect + from tests.e2e.page_objects.wizard_equipment_page import WizardEquipmentPage @@ -54,14 +56,6 @@ def test_flow_16_add_equipment_sync_mode_step(page, server_url) -> None: wiz.sync_mode.wait_for(state="visible", timeout=10_000) -def test_flow_16_add_equipment_signal_step(page, server_url) -> None: - """Completeness signal step renders the radio + sentinel filename.""" - wiz = WizardEquipmentPage(page) - _goto(page, f"{server_url}/wizard/equipment?step=signal") - wiz.signal.wait_for(state="visible", timeout=10_000) - wiz.sentinel_filename.wait_for(state="visible") - - def test_flow_16_add_equipment_review_and_confirm(page, server_url) -> None: """Review step renders Confirm; clicking it fires the success label.""" wiz = WizardEquipmentPage(page) @@ -70,3 +64,33 @@ def test_flow_16_add_equipment_review_and_confirm(page, server_url) -> None: wiz.confirm.click() page.wait_for_load_state("networkidle") wiz.success.wait_for(state="visible", timeout=5_000) + + +def test_flow_16_next_enables_on_valid_input_and_state_survives_back(page, server_url) -> None: + """An empty wizard: Next stays disabled until the identity step is + valid, then a click advances *in place* and the entered data + survives a Back step. + + Regression guard for two production bugs the seeded ``?step=`` tests + could never see: the Next button's ``disable`` was frozen at render + time (typing never re-enabled it), and the route rebuilt a fresh + empty ``EquipmentWizardState`` on every step navigation. + """ + wiz = WizardEquipmentPage(page) + _goto(page, f"{server_url}/wizard/equipment?step=identity&seed=0") + wiz.step_identity.wait_for(state="visible", timeout=10_000) + + # Bug A: Next is gated shut until the step validates. + expect(wiz.next_button).to_be_disabled() + wiz.equipment_id.fill("EQ1") + wiz.label.fill("Lab Device") + expect(wiz.next_button).to_be_enabled() + + # Advancing re-renders in place -- no page navigation. + wiz.next_button.click() + wiz.step_paths.wait_for(state="visible", timeout=10_000) + + # Bug B: stepping back keeps what the operator already typed. + wiz.back.click() + wiz.step_identity.wait_for(state="visible", timeout=10_000) + expect(wiz.equipment_id).to_have_value("EQ1") diff --git a/tests/e2e/test_flow_24_context_menus.py b/tests/e2e/test_flow_24_context_menus.py index 22ad938..87e362a 100644 --- a/tests/e2e/test_flow_24_context_menus.py +++ b/tests/e2e/test_flow_24_context_menus.py @@ -95,13 +95,59 @@ def test_flow_24_run_context_menu_items_render(page, server_url) -> None: def test_flow_24_file_list_row_context_menu(page, server_url) -> None: - """Right-clicking a file-list row surfaces Open in OS + Copy path.""" + """Right-clicking a file-list row surfaces Open in OS / Copy path / Keep local.""" _goto(page, f"{server_url}/main?view=explorer") # Select a run so the centre pane renders the seeded file list. page.locator('[data-testid="tree-node-run"]').first.click() page.wait_for_load_state("networkidle") - row = page.locator('[data-testid="file-list-row"]').first + # A non-tombstone row carries all three actions. + row = page.locator('[data-testid="file-list-row"]:not([data-tombstone])').first row.wait_for(state="visible", timeout=10_000) _open_context(page, row) page.locator('[data-testid="file-context-open-in-os"]').wait_for(state="visible", timeout=5_000) page.locator('[data-testid="file-context-copy-path"]').wait_for(state="visible", timeout=5_000) + page.locator('[data-testid="file-context-keep-local"]').wait_for(state="visible", timeout=5_000) + + +def test_flow_24_tombstone_row_has_no_open_in_os(page, server_url) -> None: + """An "On NAS" tombstone row has no local copy, so no Open-in-OS action. + + Operator-free per-file NAS sync design (2026-05-21): a cleared-run + file is listed as an "On NAS" tombstone -- it cannot be opened, but + Copy path / Keep local stay available. + """ + _goto(page, f"{server_url}/main?view=explorer") + page.locator('[data-testid="tree-node-run"]').first.click() + page.wait_for_load_state("networkidle") + tombstone = page.locator('[data-testid="file-list-row"][data-tombstone="true"]').first + tombstone.wait_for(state="visible", timeout=10_000) + _open_context(page, tombstone) + page.locator('[data-testid="file-context-copy-path"]').wait_for(state="visible", timeout=5_000) + page.locator('[data-testid="file-context-keep-local"]').wait_for(state="visible", timeout=5_000) + # The tombstone's own context menu carries no Open-in-OS item. + menu = page.locator('[data-testid="file-context-menu"]').last + assert menu.locator('[data-testid="file-context-open-in-os"]').count() == 0 + + +def test_flow_24_keep_local_badge_and_toggle(page, server_url) -> None: + """A keep-local file shows the "kept local" badge; the toggle action fires. + + Operator-free per-file NAS sync design (2026-05-21): a ``keep_local`` + file carries a small badge and its context menu offers "Don't keep + local". Clicking the keep-local item dispatches the action without + raising (the round trip to ``set_keep_local`` is covered by the unit + tests). + """ + _goto(page, f"{server_url}/main?view=explorer") + page.locator('[data-testid="tree-node-run"]').first.click() + page.wait_for_load_state("networkidle") + # The default seeded feed has exactly one keep-local file. + badge = page.locator('[data-testid="file-keep-local-badge"]') + badge.first.wait_for(state="visible", timeout=10_000) + assert badge.count() == 1 + kept_row = page.locator('[data-testid="file-list-row"][data-keep-local="true"]').first + kept_row.wait_for(state="visible", timeout=5_000) + _open_context(page, kept_row) + item = page.locator('[data-testid="file-context-keep-local"]').last + item.wait_for(state="visible", timeout=5_000) + item.click() # dispatches the keep-local toggle action without error diff --git a/tests/e2e/test_flow_26_equipment_wizard_persist.py b/tests/e2e/test_flow_26_equipment_wizard_persist.py new file mode 100644 index 0000000..51fbda5 --- /dev/null +++ b/tests/e2e/test_flow_26_equipment_wizard_persist.py @@ -0,0 +1,98 @@ +"""E2E flow 26: Add-Equipment wizard confirm against the PRODUCTION app. + +``test_flow_16`` drives the wizard's four-step *render* against the +``_test_app.py`` TestState surface. This test boots the genuine +production app (``exlab_wizard.tray._build_default_app``) under a fresh +tmp ``HOME`` and drives the wizard end to end -- type, Next, Confirm -- +then asserts the equipment was actually *persisted* to ``config.yaml``. + +Regression guard for three production bugs that flow_16 could never +see, because the test app pre-seeded a valid state and supplied its own +confirm handler: + +* the Next button's ``disable`` was frozen at render time, so typing a + valid identity never re-enabled it; +* the ``/wizard/equipment`` route rebuilt an empty wizard state on every + step navigation, discarding entered fields; +* confirm was wired to a ``deps.append_equipment`` attribute that the + production dependency factory never populated -- so confirm silently + persisted nothing. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +import pytest + +from tests.e2e._prod_server import ProdServer +from tests.e2e.conftest import PLAYWRIGHT_AVAILABLE +from tests.e2e.page_objects.wizard_equipment_page import WizardEquipmentPage + +pytestmark = pytest.mark.skipif( + not PLAYWRIGHT_AVAILABLE, + reason="playwright not installed", +) + + +@pytest.fixture +def prod_server(tmp_path: Path): + """Spawn the production wizard app under a fresh tmp HOME.""" + home = tmp_path / "home" + home.mkdir() + server = ProdServer(home) + if not server.start(): + pytest.skip("production wizard app did not become healthy within 30s") + try: + yield server + finally: + server.stop() + + +def test_equipment_wizard_confirm_persists_to_config(browser, prod_server) -> None: + """Drive the production wizard to Confirm and assert config.yaml gains the device.""" + assert not prod_server.config_path.exists(), "precondition: fresh install, no config.yaml" + + context = browser.new_context() + page = context.new_page() + try: + wiz = WizardEquipmentPage(page) + page.goto(f"{prod_server.base_url}/wizard/equipment") + page.wait_for_load_state("networkidle") + wiz.step_identity.wait_for(state="visible", timeout=10_000) + + # 1. Identity. + wiz.equipment_id.fill("MICROSCOPE_01") + wiz.label.fill("Confocal Microscope 1") + wiz.next_button.click() + + # 2. Paths. + wiz.local_root.wait_for(state="visible", timeout=10_000) + wiz.local_root.fill("/data/MICROSCOPE_01") + wiz.nas_root.fill("/srv/nas/MICROSCOPE_01") + wiz.next_button.click() + + # 3. Sync mode -- nas / rclone are the defaults. + wiz.rclone_remote.wait_for(state="visible", timeout=10_000) + wiz.rclone_remote.fill("lab-nas") + wiz.rclone_remote_path.fill("lab/MICROSCOPE_01") + wiz.next_button.click() + + # 4. Review -> Confirm. + wiz.confirm.wait_for(state="visible", timeout=10_000) + wiz.confirm.click() + page.wait_for_load_state("networkidle") + + # ``_on_confirm`` writes config.yaml synchronously before it + # navigates away, so once the click settles the file is there. + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and not prod_server.config_path.exists(): + time.sleep(0.25) + assert prod_server.config_path.exists(), "confirm must persist config.yaml" + + text = prod_server.config_path.read_text(encoding="utf-8") + assert "MICROSCOPE_01" in text + assert "Confocal Microscope 1" in text + finally: + context.close() diff --git a/tests/e2e/ux_catalog.py b/tests/e2e/ux_catalog.py index 72d45f2..add47cc 100644 --- a/tests/e2e/ux_catalog.py +++ b/tests/e2e/ux_catalog.py @@ -162,30 +162,6 @@ class UXInteraction: action="Type the equipment NAS root", outcome="Provides the EquipmentConfig.nas_root for the new entry.", ), - UXInteraction( - flow="Equipment", - route="/settings", - testid="settings-equipment-signal", - element="radio", - action="Pick the completeness signal (sentinel_file / manifest)", - outcome="Swaps the filename field between sentinel and manifest.", - ), - UXInteraction( - flow="Equipment", - route="/settings", - testid="settings-equipment-sentinel", - element="input", - action="Type the sentinel filename", - outcome="Sets the sentinel_file completeness signal filename.", - ), - UXInteraction( - flow="Equipment", - route="/settings", - testid="settings-equipment-manifest", - element="input", - action="Type the manifest filename", - outcome="Sets the manifest completeness signal filename.", - ), UXInteraction( flow="Equipment", route="/settings", @@ -565,14 +541,6 @@ class UXInteraction: action="Pick 'nas' or 'stage' sync mode", outcome="Swaps the transport sub-form between NAS-direct and stage-push.", ), - UXInteraction( - flow="Add equipment", - route="/wizard/equipment", - testid="wizard-equipment-signal", - element="radio", - action="Pick 'sentinel_file' or 'manifest' completeness signal", - outcome="Swaps the filename input between sentinel and manifest naming.", - ), UXInteraction( flow="Add equipment", route="/wizard/equipment", diff --git a/tests/fixtures/configs/complete.yaml b/tests/fixtures/configs/complete.yaml index d1fe020..ccd722f 100644 --- a/tests/fixtures/configs/complete.yaml +++ b/tests/fixtures/configs/complete.yaml @@ -22,8 +22,6 @@ equipment: label: "Confocal Microscope 1" local_root: "/data/lab" # shared equipment-first root on this workstation nas_root: "//nas01/lab" # shared equipment-first root on NAS (display value) - completeness_signal: "sentinel_file" - sentinel_filename: "acquisition_complete.flag" sync_mode: "nas" # this device syncs runs straight to the NAS (Redesign §3.2) transport: type: "rclone" # local-to-NAS transport for NASSync (§7.1) @@ -37,8 +35,6 @@ equipment: label: "Flow Cytometer 1" local_root: "/data/lab" nas_root: "/mnt/nas/lab" - completeness_signal: "manifest" - manifest_filename: "run_manifest.json" sync_mode: "nas" # this device syncs runs straight to the NAS (Redesign §3.2) transport: type: "rsync_ssh" # local-to-NAS transport for NASSync (§7.1) diff --git a/tests/fixtures/configs/incomplete_no_lims.yaml b/tests/fixtures/configs/incomplete_no_lims.yaml index 20fbc43..842edd1 100644 --- a/tests/fixtures/configs/incomplete_no_lims.yaml +++ b/tests/fixtures/configs/incomplete_no_lims.yaml @@ -17,8 +17,6 @@ equipment: label: "Confocal Microscope 1" local_root: "/data/lab" nas_root: "//nas01/lab" - completeness_signal: "sentinel_file" - sentinel_filename: "acquisition_complete.flag" sync_mode: "nas" transport: type: "rclone" diff --git a/tests/fixtures/configs/incomplete_no_paths.yaml b/tests/fixtures/configs/incomplete_no_paths.yaml index ab6d4ad..f3b6cf1 100644 --- a/tests/fixtures/configs/incomplete_no_paths.yaml +++ b/tests/fixtures/configs/incomplete_no_paths.yaml @@ -17,8 +17,6 @@ equipment: label: "Confocal Microscope 1" local_root: "/data/lab" nas_root: "//nas01/lab" - completeness_signal: "sentinel_file" - sentinel_filename: "acquisition_complete.flag" sync_mode: "nas" transport: type: "rclone" diff --git a/tests/integration/api/test_full_flow.py b/tests/integration/api/test_full_flow.py index 674359d..0c52bdb 100644 --- a/tests/integration/api/test_full_flow.py +++ b/tests/integration/api/test_full_flow.py @@ -61,8 +61,6 @@ def ready_config(tmp_path: Path) -> Config: label="Equipment 1", local_root=str(tmp_path / "data"), nas_root="/srv/nas", - completeness_signal="sentinel_file", - sentinel_filename="acquisition_complete.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/integration/controller/test_creation_flow.py b/tests/integration/controller/test_creation_flow.py index f230b4e..05cae09 100644 --- a/tests/integration/controller/test_creation_flow.py +++ b/tests/integration/controller/test_creation_flow.py @@ -78,8 +78,6 @@ def _build_config(local_root: Path, *, allowlist: list[str] | None = None) -> Co label="Equipment 1", local_root=str(local_root), nas_root="/srv/nas", - completeness_signal="sentinel_file", - sentinel_filename="acquisition_complete.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/integration/test_nas_sync.py b/tests/integration/test_nas_sync.py index c2bee86..9ec0f8c 100644 --- a/tests/integration/test_nas_sync.py +++ b/tests/integration/test_nas_sync.py @@ -77,8 +77,6 @@ def _build_config(local_root: Path) -> Config: label="Equipment 1", local_root=str(local_root), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="DONE", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -527,3 +525,166 @@ async def _hashsum(_target: Path) -> dict[str, str]: assert counter[0] == 2 finally: await client.close() + + +async def test_poller_per_file_enqueue_drives_to_synced_state( + stub_binaries_on_path: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Poller sweep -> per-file enqueue -> drive -> verify -> sync_state.json + records ``synced_signature`` + ``verified_at`` for the synced files. + + Exercises the full operator-free per-file NAS sync path end-to-end: + the :class:`QuiescenceSyncPoller` discovers the run, computes the + eligible file list, and feeds it to a real :class:`NASSyncClient` that + drives the job through the stub rclone transport + verifier. + """ + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + from exlab_wizard.orchestrator.quiescence_poller import QuiescenceSyncPoller + + local_root = tmp_path / "local" + local_root.mkdir() + nas_root = tmp_path / "nas" + monkeypatch.setenv("STUB_RCLONE_BEHAVIOR", "success") + monkeypatch.setenv("STUB_RCLONE_DEST_ROOT", str(nas_root)) + + cfg = _build_config(local_root) + run_dir = await _populate_run(local_root) + writer = CreationWriter(lock_timeout_seconds=10.0) + sync_state = SyncStateWriter() + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + sync_state_writer=sync_state, + worker_poll_interval_s=0.01, + ) + await client.init() + poller = QuiescenceSyncPoller( + config=cfg, + nas_sync=client, + sync_state_writer=sync_state, + ) + try: + # First sweep observes the file; second sweep (past the settle + # window) finds it quiet and enqueues the per-file subset. + assert await poller.poll_once(now_monotonic=0.0) == [] + enqueued = await poller.poll_once(now_monotonic=cfg.sync.quiescence_minutes * 60 + 1.0) + assert enqueued == [run_dir] + + # The worker drives the per-file job through to VERIFIED. + async def _by_run_path(_ignored: str) -> SyncJobRow | None: + return await client._queue.get_by_run_path(run_dir) + + row = await _wait_for_state( + _by_run_path, + "", + {SyncJobState.VERIFIED, SyncJobState.CLEANUP_ELIGIBLE, SyncJobState.CLEANED}, + ) + assert row.state in { + SyncJobState.VERIFIED, + SyncJobState.CLEANUP_ELIGIBLE, + SyncJobState.CLEANED, + } + assert row.files == ("data.bin",) + + # sync_state.json records the verified file. + state = await sync_state.read(run_dir) + assert "data.bin" in state.files + assert state.files["data.bin"].synced_signature is not None + assert state.files["data.bin"].verified_at is not None + finally: + await client.close() + + +async def test_poller_to_cleanup_honors_keep_local_and_stamps_cleared( + stub_binaries_on_path: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Full Phase 5 path: poller sweep -> enqueue -> verify -> SYNCED rollup + -> cleanup runs, keeping a ``keep_local`` file and stamping ``cleared_at``. + + Exercises the operator-free per-file NAS sync cleanup contract + end-to-end with a real :class:`NASSyncClient` over the stub rclone + transport: the run carries two data files, one flagged ``keep_local``; + after cleanup the kept file survives, the other is removed, the + ``.exlab-wizard/`` metadata subtree is retained, and the run's + ``sync_state.json`` rolls up to ``CLEARED``. + """ + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + from exlab_wizard.constants import RunSyncState + from exlab_wizard.orchestrator.quiescence_poller import QuiescenceSyncPoller + + local_root = tmp_path / "local" + local_root.mkdir() + nas_root = tmp_path / "nas" + monkeypatch.setenv("STUB_RCLONE_BEHAVIOR", "success") + monkeypatch.setenv("STUB_RCLONE_DEST_ROOT", str(nas_root)) + + # ``min_verify_passes=1`` + ``min_age_hours=0`` so cleanup runs in the + # same worker pass that promotes the job to VERIFIED. + cfg = _build_config(local_root) + run_dir = local_root / "EQ1" / "PROJ-0042" / "Runs" / "Run_2026-04-17T14-32-00" + run_dir.mkdir(parents=True) + (run_dir / "data.bin").write_bytes(b"payload-bytes") + (run_dir / "keep.bin").write_bytes(b"keep-me-local") + cache = run_dir / CACHE_DIR_NAME + cache.mkdir() + (cache / CREATION_JSON_NAME).write_bytes(msgspec_json.encode(_make_creation(run_dir))) + + writer = CreationWriter(lock_timeout_seconds=10.0) + sync_state = SyncStateWriter() + # Operator flags one file keep-local before the sync runs. + await sync_state.set_keep_local(run_dir, "keep.bin", True) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + sync_state_writer=sync_state, + worker_poll_interval_s=0.01, + ) + await client.init() + poller = QuiescenceSyncPoller( + config=cfg, + nas_sync=client, + sync_state_writer=sync_state, + ) + try: + # Poller discovers the run and enqueues its quiet files past the + # settle window. + assert await poller.poll_once(now_monotonic=0.0) == [] + enqueued = await poller.poll_once(now_monotonic=cfg.sync.quiescence_minutes * 60 + 1.0) + assert enqueued == [run_dir] + + async def _by_run_path(_ignored: str) -> SyncJobRow | None: + return await client._queue.get_by_run_path(run_dir) + + # The worker drives the job through verify into the cleanup states. + await _wait_for_state( + _by_run_path, + "", + {SyncJobState.CLEANED}, + ) + + # The keep_local file survives; the other data file is removed. + assert (run_dir / "keep.bin").exists() + assert not (run_dir / "data.bin").exists() + # The metadata subtree is retained so tombstones still render. + assert cache.exists() + + # sync_state.json rolled up to CLEARED (cleared_at stamped). + state = await sync_state.read(run_dir) + assert state.cleared_at is not None + assert SyncStateWriter.rollup_state(state) is RunSyncState.CLEARED + # Both files were credited as verified before cleanup ran. + assert state.files["data.bin"].verified_at is not None + assert state.files["keep.bin"].verified_at is not None + assert state.files["keep.bin"].keep_local is True + finally: + await client.close() diff --git a/tests/integration/test_orchestrator_lifecycle.py b/tests/integration/test_orchestrator_lifecycle.py index 83b1e1c..7961c59 100644 --- a/tests/integration/test_orchestrator_lifecycle.py +++ b/tests/integration/test_orchestrator_lifecycle.py @@ -1,32 +1,28 @@ -"""Integration test for the full orchestrator staging lifecycle. - -Backend Spec §12, §13. Drives a single run through the entire five-state -pipeline from staging -> complete -> sync_queued -> sync_verified -> -cleared via the ``StagingWatcher`` + a stub NAS sync client + the -:class:`IngestWriter`. Asserts that: - -* The on-disk ``ingest.json`` ends with all five state entries. -* ``cleanup_eligible`` returns False under manual mode and True under - scheduled mode after the retain window elapses. -* The ``GET /staging`` and ``POST /staging/.../clear`` endpoints surface - the run end-to-end. +"""Integration test for the orchestrator quiescence-sync lifecycle. + +Backend Spec §12, §13; operator-free per-file NAS sync design (2026-05-21). +Drives a run through the quiescence poller -> sync-queue enqueue -> the +``GET /staging`` + ``POST /staging/.../clear`` endpoints. Asserts that: + +* the :class:`QuiescenceSyncPoller` enqueues a run only once its files + have settled for ``sync.quiescence_minutes``; +* a quiet file already synced at its current signature (recorded in + ``sync_state.json``) is not re-enqueued; +* the ``GET /staging`` endpoint surfaces the run with the queue-derived + ``current_state``, and ``POST /staging/.../clear`` deletes a verified + run's staging copy. """ from __future__ import annotations +import asyncio from dataclasses import dataclass -from datetime import UTC, datetime, timedelta from pathlib import Path -import msgspec from fastapi.testclient import TestClient from exlab_wizard.api import AppDependencies, create_app -from exlab_wizard.api.schemas import ( - CreationJson, - IngestJson, -) -from exlab_wizard.cache.ingest_writer import IngestWriter +from exlab_wizard.cache.sync_state_writer import SyncStateWriter from exlab_wizard.config.models import ( BandwidthConfig, Config, @@ -35,16 +31,10 @@ OrchestratorStagingCleanup, PathsConfig, RcloneTransport, + SyncConfig, ) -from exlab_wizard.constants import ( - CACHE_DIR_NAME, - CREATION_JSON_NAME, - CREATION_JSON_VERSION, - INGEST_JSON_NAME, - IngestState, - StagingCleanupMode, -) -from exlab_wizard.orchestrator.staging_watcher import StagingWatcher +from exlab_wizard.constants import RUNS_DIR_NAME +from exlab_wizard.orchestrator.quiescence_poller import QuiescenceSyncPoller @dataclass @@ -54,66 +44,37 @@ class _Handle: run_path: str = "" +@dataclass +class _StubJobRow: + run_path: str + state: str + + class _StubNasSync: - """In-memory sync client that records enqueue + status calls.""" + """In-memory sync client that records per-file enqueue + serves status/list_all.""" def __init__(self) -> None: - self.enqueued: list[Path] = [] + self.enqueued: list[tuple[Path, list[str]]] = [] self.status_responses: dict[str, str] = {} - async def enqueue(self, run_path: Path) -> _Handle: - self.enqueued.append(run_path) + async def enqueue(self, run_path: Path, files: list[str] | None = None) -> _Handle: + self.enqueued.append((run_path, list(files or []))) return _Handle(run_path=str(run_path)) async def status(self, run_path: Path) -> str: - return self.status_responses.get(str(run_path), "queued") - - -class _StubCreationCache: - def __init__(self, *, payload: CreationJson | None = None) -> None: - self._payload = payload - - async def read_creation_snapshot(self, path: Path) -> CreationJson: - if self._payload is None: - raise FileNotFoundError(path) - return self._payload - - -def _make_creation() -> CreationJson: - return msgspec.convert( - { - "schema_version": CREATION_JSON_VERSION, - "created_at": "2026-04-17T14:32:00Z", - "created_by": "asmith", - "level": "run", - "run_kind": "experimental", - "lims_project": { - "uid": "abc", - "short_id": "PROJ-0001", - "name_at_creation": "Test Project", - }, - "template": { - "name": "confocal_run", - "version": "1.0", - "source_path": "templates/confocal_run", - "run_scope": "experimental", - }, - "variables": {}, - "paths": { - "local": "/staging/EQ1/PROJ-0001/Run_2026-04-17T14-32-00", - "nas": "/nas/EQ1/PROJ-0001/Run_2026-04-17T14-32-00", - }, - }, - type=CreationJson, - ) + return self.status_responses.get(str(run_path), "none") + async def list_all(self) -> list[_StubJobRow]: + return [ + _StubJobRow(run_path=path, state=state) for path, state in self.status_responses.items() + ] -def _make_config( - staging_root: Path, - *, - cleanup_mode: str = StagingCleanupMode.SCHEDULED.value, - retain_hours: int = 1, -) -> Config: + @property + def enqueued_paths(self) -> list[Path]: + return [run_path for run_path, _ in self.enqueued] + + +def _make_config(staging_root: Path, *, quiescence_minutes: int = 1) -> Config: return Config( paths=PathsConfig(local_root=str(staging_root)), equipment=[ @@ -122,8 +83,6 @@ def _make_config( label="Equipment 1", local_root=str(staging_root), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="run_complete", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -135,224 +94,127 @@ def _make_config( orchestrator=OrchestratorConfig( label="ORCH", staging_root=str(staging_root), - staging_cleanup=OrchestratorStagingCleanup( - mode=cleanup_mode, retain_hours=retain_hours - ), + staging_cleanup=OrchestratorStagingCleanup(), ), + sync=SyncConfig(quiescence_minutes=quiescence_minutes), ) -def _stage_pushed_run(staging_root: Path) -> Path: - run_dir = staging_root / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" +def _stage_run(staging_root: Path) -> Path: + run_dir = staging_root / "EQ1" / "PROJ-0001" / RUNS_DIR_NAME / "Run_2026-04-17T14-32-00" run_dir.mkdir(parents=True) (run_dir / "data.bin").write_bytes(b"hello-world" * 100) - cache = run_dir / CACHE_DIR_NAME - cache.mkdir() - (cache / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(_make_creation())) return run_dir # --------------------------------------------------------------------------- -# End-to-end lifecycle +# Quiescence poller end-to-end # --------------------------------------------------------------------------- -async def test_orchestrator_full_five_state_lifecycle(tmp_path: Path) -> None: - """Drive a single run end-to-end through every state transition.""" - config = _make_config(tmp_path) - run_dir = _stage_pushed_run(tmp_path) +async def test_poller_enqueues_run_after_settle_window(tmp_path: Path) -> None: + """A staged run is enqueued only once its files settle for the window.""" + config = _make_config(tmp_path, quiescence_minutes=1) # 60s window + run_dir = _stage_run(tmp_path) nas_sync = _StubNasSync() - ingest_writer = IngestWriter() - watcher = StagingWatcher( - config=config, - ingest_writer=ingest_writer, - nas_sync=nas_sync, - cache_creation=_StubCreationCache(payload=_make_creation()), - poll_interval_s=0.01, + poller = QuiescenceSyncPoller( + config=config, nas_sync=nas_sync, sync_state_writer=SyncStateWriter() ) - # 1. Bootstrap -> staging. - await watcher.evaluate_run(run_dir) - ingest_path = run_dir / CACHE_DIR_NAME / INGEST_JSON_NAME - state = await ingest_writer.read_ingest(ingest_path) - assert state.current_state == IngestState.STAGING.value - - # 2. staging -> complete (sentinel landed). - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - state = await ingest_writer.read_ingest(ingest_path) - assert state.current_state == IngestState.COMPLETE.value - complete_entry = state.history[-1] - assert complete_entry["files_received"] >= 1 - assert complete_entry["bytes_received"] >= len(b"hello-world" * 100) - - # 3. complete -> sync_queued. - await watcher.evaluate_run(run_dir) - state = await ingest_writer.read_ingest(ingest_path) - assert state.current_state == IngestState.SYNC_QUEUED.value - assert nas_sync.enqueued == [run_dir] - - # 4. sync_queued -> sync_verified. - nas_sync.status_responses[str(run_dir)] = "verified" - await watcher.evaluate_run(run_dir) - state = await ingest_writer.read_ingest(ingest_path) - assert state.current_state == IngestState.SYNC_VERIFIED.value - - # 5. sync_verified -> cleared (after backdating the entry). - payload = msgspec.json.decode(ingest_path.read_bytes(), type=IngestJson) - backdated_at = (datetime.now(tz=UTC) - timedelta(hours=2)).strftime("%Y-%m-%dT%H:%M:%SZ") - new_history = [] - for entry in payload.history: - new_entry = dict(entry) - if entry.get("state") == IngestState.SYNC_VERIFIED.value: - new_entry["at"] = backdated_at - new_history.append(new_entry) - new_payload = msgspec.structs.replace(payload, history=new_history) - ingest_path.write_bytes(msgspec.json.encode(new_payload)) - - final_state = await watcher.evaluate_run(run_dir) - assert final_state == IngestState.CLEARED - # The directory has been removed; the in-memory sequence shows every - # transition recorded before the dir went away. - assert not run_dir.exists() - - -async def test_orchestrator_lifecycle_through_api(tmp_path: Path) -> None: - """Mount the full app and drive the staging endpoints against a seeded run.""" - config = _make_config(tmp_path, cleanup_mode=StagingCleanupMode.MANUAL.value) - run_dir = _stage_pushed_run(tmp_path) - nas_sync = _StubNasSync() - ingest_writer = IngestWriter() - - # Walk the watcher up to sync_verified manually. - watcher = StagingWatcher( - config=config, - ingest_writer=ingest_writer, - nas_sync=nas_sync, - cache_creation=_StubCreationCache(payload=_make_creation()), + # First observation -- not yet quiet. + assert await poller.poll_once(now_monotonic=0.0) == [] + # Still inside the window. + assert await poller.poll_once(now_monotonic=45.0) == [] + # Window elapsed -> enqueued exactly once, carrying the quiet file. + assert await poller.poll_once(now_monotonic=60.0) == [run_dir] + assert nas_sync.enqueued == [(run_dir, ["data.bin"])] + + +async def test_poller_skips_file_already_synced_at_current_signature(tmp_path: Path) -> None: + """A quiet file recorded in sync_state.json at its current signature + is not re-enqueued; the run drops out of the eligible set.""" + config = _make_config(tmp_path, quiescence_minutes=1) + run_dir = _stage_run(tmp_path) + writer = SyncStateWriter() + target = run_dir / "data.bin" + st = target.stat() + await writer.upsert_file( + run_dir, + "data.bin", + synced_signature=(st.st_size, st.st_mtime_ns), + verified_at="2026-04-17T15:00:00Z", ) - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - await watcher.evaluate_run(run_dir) - nas_sync.status_responses[str(run_dir)] = "verified" - await watcher.evaluate_run(run_dir) + nas_sync = _StubNasSync() + poller = QuiescenceSyncPoller(config=config, nas_sync=nas_sync, sync_state_writer=writer) - deps = AppDependencies( - config=config, - nas_sync=nas_sync, - ingest_writer=ingest_writer, - ) - app = create_app(dependencies=deps) - with TestClient(app) as client: - # GET /staging surfaces the run with its current sync_verified state. - resp = client.get("/api/v1/staging") - assert resp.status_code == 200 - rows = resp.json()["runs"] - assert len(rows) == 1 - assert rows[0]["current_state"] == IngestState.SYNC_VERIFIED.value + await poller.poll_once(now_monotonic=0.0) + assert await poller.poll_once(now_monotonic=120.0) == [] + assert nas_sync.enqueued == [] - # POST /staging/{run}/clear deletes the run. - resp = client.post(f"/api/v1/staging/{run_dir}/clear") - assert resp.status_code == 200 - body = resp.json() - assert body["files_freed"] >= 1 - assert body["bytes_freed"] >= 1 - assert not run_dir.exists() +# --------------------------------------------------------------------------- +# Staging API end-to-end +# --------------------------------------------------------------------------- -async def test_orchestrator_concurrent_pushes_are_independent(tmp_path: Path) -> None: - """Two equipment pushes don't interfere with each other's lifecycles.""" - config = _make_config(tmp_path) - config.equipment.append( - EquipmentConfig( - id="EQ2", - label="Equipment 2", - local_root=str(tmp_path), - nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="run_complete", - transport=RcloneTransport( - type="rclone", - rclone_remote="lab-nas", - rclone_remote_path="/srv/nas2", - ), - ), +def _mark_run_synced(run_dir: Path, writer: SyncStateWriter) -> None: + """Write a fully-verified ``sync_state.json`` so the run rolls up to ``synced``.""" + asyncio.run( + writer.upsert_file( + run_dir, "data.bin", synced_signature=(1100, 1), verified_at="2026-05-21T00:00:00Z" + ) ) - run1 = _stage_pushed_run(tmp_path) - run2 = tmp_path / "EQ2" / "PROJ-0002" / "Run_2026-04-18T00-00-00" - run2.mkdir(parents=True) - (run2 / "data.bin").write_bytes(b"second-equipment") - cache2 = run2 / CACHE_DIR_NAME - cache2.mkdir() - (cache2 / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(_make_creation())) - nas_sync = _StubNasSync() - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=nas_sync, - cache_creation=_StubCreationCache(payload=_make_creation()), - ) - # Bootstrap both runs. - states = await watcher.poll_once() - assert states.count(IngestState.STAGING) == 2 - # Advance only run1 to complete. - (run1 / "run_complete").write_text("done") - states = await watcher.poll_once() - # poll_once visits both, but only run1 has the sentinel. - assert IngestState.COMPLETE in states - assert IngestState.STAGING in states +def test_staging_endpoint_surfaces_run_with_rollup_state(tmp_path: Path) -> None: + """``GET /staging`` reports the run with its ``sync_state.json`` rollup.""" + config = _make_config(tmp_path) + run_dir = _stage_run(tmp_path) + writer = SyncStateWriter() + _mark_run_synced(run_dir, writer) + deps = AppDependencies(config=config, nas_sync=_StubNasSync(), sync_state_writer=writer) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.get("/api/v1/staging") + assert resp.status_code == 200 + runs = resp.json()["runs"] + assert len(runs) == 1 + assert runs[0]["path"] == str(run_dir) + assert runs[0]["current_state"] == "synced" + +def test_clear_endpoint_deletes_synced_run(tmp_path: Path) -> None: + """``POST /staging/.../clear`` deletes a fully-``synced`` run's data files, + retains the metadata subtree, and stamps ``cleared_at``.""" + from exlab_wizard.constants import CACHE_DIR_NAME -async def test_orchestrator_endpoint_shows_test_run_runs(tmp_path: Path) -> None: - """Test runs (TestRuns/TestRun_) surface alongside experimental ones.""" config = _make_config(tmp_path) - test_run_dir = tmp_path / "EQ1" / "PROJ-0001" / "TestRuns" / "TestRun_2026-04-17T09-12-00" - test_run_dir.mkdir(parents=True) - (test_run_dir / "data.bin").write_bytes(b"x" * 10) - cache = test_run_dir / CACHE_DIR_NAME - cache.mkdir() - test_creation = msgspec.convert( - { - "schema_version": CREATION_JSON_VERSION, - "created_at": "2026-04-17T09:12:00Z", - "created_by": "asmith", - "level": "run", - "run_kind": "test", - "lims_project": { - "uid": "abc", - "short_id": "PROJ-0001", - "name_at_creation": "Test Project", - }, - "template": { - "name": "confocal_run", - "version": "1.0", - "source_path": "templates/confocal_run", - "run_scope": "test", - }, - "variables": {}, - "paths": {"local": str(test_run_dir), "nas": str(test_run_dir)}, - }, - type=CreationJson, - ) - (cache / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(test_creation)) + run_dir = _stage_run(tmp_path) + writer = SyncStateWriter() + _mark_run_synced(run_dir, writer) + deps = AppDependencies(config=config, nas_sync=_StubNasSync(), sync_state_writer=writer) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post(f"/api/v1/staging/{run_dir}/clear") + assert resp.status_code == 200 + assert resp.json()["files_freed"] >= 1 + # Data file gone; the .exlab-wizard/ metadata subtree survives so the + # cleared run still renders "On NAS" tombstones. + assert not (run_dir / "data.bin").exists() + assert (run_dir / CACHE_DIR_NAME).exists() + state = asyncio.run(writer.read(run_dir)) + assert state.cleared_at is not None - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=_StubNasSync(), - cache_creation=_StubCreationCache(payload=test_creation), - ) - await watcher.evaluate_run(test_run_dir) - deps = AppDependencies(config=config) +def test_clear_endpoint_rejects_unsynced_run(tmp_path: Path) -> None: + """A run that is not fully ``synced`` cannot be cleared.""" + config = _make_config(tmp_path) + run_dir = _stage_run(tmp_path) # no sync_state.json -> rollup "syncing" + deps = AppDependencies( + config=config, nas_sync=_StubNasSync(), sync_state_writer=SyncStateWriter() + ) app = create_app(dependencies=deps) with TestClient(app) as client: - resp = client.get("/api/v1/staging") - assert resp.status_code == 200 - rows = resp.json()["runs"] - assert len(rows) == 1 - assert rows[0]["run_kind"] == "test" + resp = client.post(f"/api/v1/staging/{run_dir}/clear") + assert resp.status_code == 409 + assert run_dir.exists() diff --git a/tests/integration/test_schema_major_mismatch.py b/tests/integration/test_schema_major_mismatch.py index 2d73ede..9c49717 100644 --- a/tests/integration/test_schema_major_mismatch.py +++ b/tests/integration/test_schema_major_mismatch.py @@ -1,8 +1,8 @@ """Integration tests for the cross-major-read-fails contract from §11.9.2. For every cache file written by the wizard (``creation.json``, -``readme_fields.json``, ``equipment.json``, ``test_runs.json``, -``ingest.json``), the reader MUST refuse a file whose ``schema_version`` +``readme_fields.json``, ``equipment.json``, ``test_runs.json``), the +reader MUST refuse a file whose ``schema_version`` major component is different from the reader's. The error must be a ``SchemaMajorMismatchError`` with ``expected_major == 1`` (every cache schema is currently major 1) and ``found`` mirroring the on-disk string @@ -150,22 +150,6 @@ class _CacheCase: "equipment": "CONFOCAL_01", }, ), - _CacheCase( - label="ingest_json", - candidate_modules=("exlab_wizard.cache.ingest_writer",), - candidate_classes=("IngestWriter",), - candidate_readers=("read_ingest",), - payload_v2={ - "schema_version": "2.0", - "project_name": "Cortex Q3 Pilot", - "equipment_id": "CONFOCAL_01", - "run_kind": "experimental", - "run_path": "CONFOCAL_01/PROJ-0042/Run_x", - "transport": "smb_mount", - "current_state": "staging", - "history": [], - }, - ), ) diff --git a/tests/unit/api/test_browse.py b/tests/unit/api/test_browse.py index 048a8a6..ecf1259 100644 --- a/tests/unit/api/test_browse.py +++ b/tests/unit/api/test_browse.py @@ -29,6 +29,7 @@ CREATION_JSON_VERSION, README_FILE_NAME, RUN_DIR_PREFIX, + RunSyncState, SyncStatus, ) @@ -46,8 +47,6 @@ def _config_with_local_root(local_root: Path) -> Config: label="Equipment 1", local_root=str(local_root), nas_root="/srv/nas", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -107,7 +106,10 @@ def test_get_tree_lists_equipment_and_projects(tmp_path: Path) -> None: assert project["name"] == "Cortex Q3 Pilot" assert len(project["runs"]) == 1 assert project["runs"][0]["kind"] == "experimental" - assert project["runs"][0]["sync_status"] == SyncStatus.PENDING.value + # Operator-free per-file NAS sync design (2026-05-21): the run-node + # rollup is derived from sync_state.json. A run with no sync_state.json + # yet rolls up to ``syncing`` (nothing tracked / verified). + assert project["runs"][0]["sync_status"] == RunSyncState.SYNCING.value def test_get_tree_returns_empty_when_no_equipment(tmp_path: Path) -> None: @@ -302,117 +304,113 @@ def test_get_folder_rejects_path_outside_configured_roots(tmp_path: Path) -> Non # --------------------------------------------------------------------------- -def _write_ingest_json( - run_dir: Path, - *, - current_state: str, - history: list[dict], -) -> None: - from exlab_wizard.api.schemas import IngestJson - from exlab_wizard.cache.ingest_writer import IngestWriter - from exlab_wizard.constants import INGEST_JSON_NAME, INGEST_JSON_VERSION +class _StubJobRow: + """A minimal sync-queue job row for the run-log tests.""" - cache = run_dir / CACHE_DIR_NAME - cache.mkdir(parents=True, exist_ok=True) - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "PROJ-0001", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": str(run_dir), - "transport": "smb_mount", - "current_state": current_state, - "history": history, - }, - type=IngestJson, - ) - # Use synchronous write since the IngestWriter's async API requires - # an event loop here; msgspec encoded payload + write_bytes is fine - # for test fixtures. - _ = IngestWriter # keep import for type/intent clarity - (cache / INGEST_JSON_NAME).write_bytes(msgspec.json.encode(payload)) + def __init__(self, *, run_path: str, state: str) -> None: + from exlab_wizard.sync.queue import SyncJobState + + self.run_path = run_path + self.state = SyncJobState(state) + self.attempts = 2 + self.verify_passes = 1 + self.last_error: str | None = None + self.nas_path: str | None = None + self.verified_at = "2026-05-01T10:35:00Z" + self.enqueued_at = "2026-05-01T10:00:00Z" + + +class _StubNasSync: + """Sync-queue stub serving ``get_by_run_path`` for the log endpoint.""" + + def __init__(self, row: _StubJobRow | None = None) -> None: + self._row = row + async def get_by_run_path(self, run_path: Path) -> _StubJobRow | None: + if self._row is None or self._row.run_path != str(run_path): + return None + return self._row -def test_get_run_log_returns_history_entries(tmp_path: Path) -> None: - """A staged run's ingest.json history is surfaced as the log.""" - from exlab_wizard.constants import IngestState +def test_get_run_log_returns_queue_derived_history(tmp_path: Path) -> None: + """A staged run's sync-queue job state is surfaced as the log.""" run_dir = tmp_path / "data" / "EQ1" / "PROJ-0001" / "Run_2026-05-01T10-00-00" run_dir.mkdir(parents=True) - _write_ingest_json( - run_dir, - current_state=IngestState.SYNC_VERIFIED.value, - history=[ - { - "state": IngestState.STAGING.value, - "at": "2026-05-01T10:00:00Z", - "host": "h1", - }, - { - "state": IngestState.COMPLETE.value, - "at": "2026-05-01T10:30:00Z", - "host": "h1", - "files_received": 12, - }, - { - "state": IngestState.SYNC_QUEUED.value, - "at": "2026-05-01T10:31:00Z", - "host": "h1", - }, - { - "state": IngestState.SYNC_VERIFIED.value, - "at": "2026-05-01T10:35:00Z", - "host": "h1", - }, - ], + nas_sync = _StubNasSync(_StubJobRow(run_path=str(run_dir), state="verified")) + deps = AppDependencies( + config=_config_with_local_root(tmp_path / "data"), + nas_sync=nas_sync, ) - deps = AppDependencies(config=_config_with_local_root(tmp_path / "data")) app = create_app(dependencies=deps) client = TestClient(app) resp = client.get(f"/api/v1/run/{run_dir}/log") assert resp.status_code == 200 body = resp.json() assert body["path"] == str(run_dir) - assert body["current_state"] == IngestState.SYNC_VERIFIED.value - assert len(body["history"]) == 4 - states = [entry["state"] for entry in body["history"]] - assert states == [ - IngestState.STAGING.value, - IngestState.COMPLETE.value, - IngestState.SYNC_QUEUED.value, - IngestState.SYNC_VERIFIED.value, - ] - # Extra ingest fields (e.g. files_received) come through as payload. - assert body["history"][1]["payload"] == {"files_received": 12} - - -def test_get_run_log_404_when_ingest_missing(tmp_path: Path) -> None: - """A run without an ingest.json returns 404 ``ingest_not_found``.""" - deps = AppDependencies(config=_config_with_local_root(tmp_path)) + assert body["current_state"] == "verified" + assert len(body["history"]) == 1 + assert body["history"][0]["state"] == "verified" + # Queue extras (attempts, verify_passes) come through as payload. + assert body["history"][0]["payload"]["attempts"] == 2 + + +def test_get_run_log_forwards_failure_extras_in_payload(tmp_path: Path) -> None: + """A failed job row's ``last_error`` / ``attempts`` / ``verify_passes`` / + ``nas_path`` are all forwarded into the log entry's free-form payload. + + Directly exercises the ``extras`` extraction in + :func:`browse._run_log_from_queue` -- the baseline test leaves + ``last_error`` / ``nas_path`` unset, so this asserts the truthy-only + extraction picks up every populated extra. + """ + run_dir = tmp_path / "data" / "EQ1" / "PROJ-0001" / "Run_2026-05-02T08-00-00" + run_dir.mkdir(parents=True) + row = _StubJobRow(run_path=str(run_dir), state="failed") + row.attempts = 4 + row.verify_passes = 0 + row.last_error = "transport timeout" + row.nas_path = "/srv/nas/EQ1/run" + deps = AppDependencies( + config=_config_with_local_root(tmp_path / "data"), + nas_sync=_StubNasSync(row), + ) app = create_app(dependencies=deps) client = TestClient(app) - resp = client.get(f"/api/v1/run/{tmp_path}/nope/log") - assert resp.status_code == 404 - # Reuses the existing ``session_not_found`` code -- same allowlist as - # the run-detail endpoint (creation.json missing vs ingest.json - # missing share semantics: the run record is unreadable). - assert resp.json()["error"]["code"] == "session_not_found" + resp = client.get(f"/api/v1/run/{run_dir}/log") + assert resp.status_code == 200 + payload = resp.json()["history"][0]["payload"] + assert payload["attempts"] == 4 + assert payload["last_error"] == "transport timeout" + assert payload["nas_path"] == "/srv/nas/EQ1/run" + # ``verify_passes`` is 0 (falsy) -- the truthy-only extraction omits it. + assert "verify_passes" not in payload -def test_get_run_log_422_when_ingest_malformed(tmp_path: Path) -> None: - """A corrupt ingest.json surfaces 422 from msgspec.""" +def test_get_run_log_empty_history_when_no_queue_job(tmp_path: Path) -> None: + """A run with no sync-queue job returns an empty history + 'none' state.""" run_dir = tmp_path / "data" / "EQ1" / "PROJ-0001" / "Run_x" - cache = run_dir / CACHE_DIR_NAME - cache.mkdir(parents=True) - from exlab_wizard.constants import INGEST_JSON_NAME - - (cache / INGEST_JSON_NAME).write_bytes(b"{not-valid-json") - deps = AppDependencies(config=_config_with_local_root(tmp_path / "data")) + run_dir.mkdir(parents=True) + deps = AppDependencies( + config=_config_with_local_root(tmp_path / "data"), + nas_sync=_StubNasSync(), + ) app = create_app(dependencies=deps) client = TestClient(app) resp = client.get(f"/api/v1/run/{run_dir}/log") - assert resp.status_code == 422 + assert resp.status_code == 200 + body = resp.json() + assert body["current_state"] == "none" + assert body["history"] == [] + + +def test_get_run_log_404_when_run_missing(tmp_path: Path) -> None: + """A run directory that does not exist returns 404 ``session_not_found``.""" + deps = AppDependencies(config=_config_with_local_root(tmp_path)) + app = create_app(dependencies=deps) + client = TestClient(app) + resp = client.get(f"/api/v1/run/{tmp_path}/nope/log") + assert resp.status_code == 404 + assert resp.json()["error"]["code"] == "session_not_found" # --------------------------------------------------------------------------- @@ -493,3 +491,164 @@ def test_scan_folder_sync_raises_404_for_missing_path(tmp_path: Path) -> None: assert exc.status_code == 404 else: # pragma: no cover -- defensive raise AssertionError("expected HTTPException") + + +# --------------------------------------------------------------------------- +# Per-file sync status from sync_state.json + cleared-run tombstones +# (operator-free per-file NAS sync design, 2026-05-21) +# --------------------------------------------------------------------------- + + +def _seed_run_with_creation(local_root: Path) -> Path: + """Create a run dir with a creation.json so it is recognised as a run.""" + eq_dir = local_root / "EQ1" + run_dir = eq_dir / "Cortex" / f"{RUN_DIR_PREFIX}2026-05-21T00-00-00" + run_dir.mkdir(parents=True) + _write_creation_json(run_dir) + return run_dir + + +def _write_sync_state(run_dir: Path, files: dict, *, cleared: bool = False) -> None: + """Write a sync_state.json with the given per-file records.""" + import asyncio + + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + writer = SyncStateWriter() + for rel, rec in files.items(): + asyncio.run( + writer.upsert_file( + run_dir, + rel, + synced_signature=rec.get("synced_signature"), + verified_at=rec.get("verified_at"), + ) + ) + if rec.get("keep_local"): + asyncio.run(writer.set_keep_local(run_dir, rel, True)) + if cleared: + asyncio.run(writer.mark_cleared(run_dir)) + + +def test_per_file_status_acquiring_when_not_in_sync_state(tmp_path: Path) -> None: + """A file on disk but absent from sync_state.json reads as ``acquiring``.""" + from exlab_wizard.api.routers import browse + + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + (run_dir / "scan.tif").write_bytes(b"x" * 10) + config = _config_with_local_root(local_root) + resp = browse.scan_folder_sync(str(run_dir), config) + by_name = {e.name: e for e in resp.entries} + assert by_name["scan.tif"].sync_status == "acquiring" + + +def test_per_file_status_syncing_when_unverified(tmp_path: Path) -> None: + """A recorded file with verified_at null + on disk reads as ``syncing``.""" + from exlab_wizard.api.routers import browse + + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + (run_dir / "scan.tif").write_bytes(b"x" * 10) + _write_sync_state(run_dir, {"scan.tif": {"synced_signature": (10, 1)}}) + config = _config_with_local_root(local_root) + resp = browse.scan_folder_sync(str(run_dir), config) + by_name = {e.name: e for e in resp.entries} + assert by_name["scan.tif"].sync_status == "syncing" + + +def test_per_file_status_synced_when_verified_and_on_disk(tmp_path: Path) -> None: + """A recorded+verified file still on disk reads as ``synced``.""" + from exlab_wizard.api.routers import browse + + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + (run_dir / "scan.tif").write_bytes(b"x" * 10) + _write_sync_state( + run_dir, + {"scan.tif": {"synced_signature": (10, 1), "verified_at": "2026-05-21T01:00:00Z"}}, + ) + config = _config_with_local_root(local_root) + resp = browse.scan_folder_sync(str(run_dir), config) + by_name = {e.name: e for e in resp.entries} + assert by_name["scan.tif"].sync_status == "synced" + + +def test_per_file_status_on_nas_tombstone_for_cleared_run(tmp_path: Path) -> None: + """A cleared run lists verified-but-absent files as ``on_nas`` tombstones.""" + from exlab_wizard.api.routers import browse + + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + # File verified, then cleared from disk -- only sync_state.json remains. + _write_sync_state( + run_dir, + {"scan.tif": {"synced_signature": (10, 1), "verified_at": "2026-05-21T01:00:00Z"}}, + cleared=True, + ) + config = _config_with_local_root(local_root) + resp = browse.scan_folder_sync(str(run_dir), config) + by_name = {e.name: e for e in resp.entries} + assert "scan.tif" in by_name + tomb = by_name["scan.tif"] + assert tomb.sync_status == "on_nas" + assert tomb.tombstone is True + assert tomb.is_dir is False + assert tomb.size_bytes is None + + +def test_per_file_status_keep_local_flag_carried(tmp_path: Path) -> None: + """A keep_local file carries the flag alongside its sync status.""" + from exlab_wizard.api.routers import browse + + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + (run_dir / "scan.tif").write_bytes(b"x" * 10) + _write_sync_state( + run_dir, + { + "scan.tif": { + "synced_signature": (10, 1), + "verified_at": "2026-05-21T01:00:00Z", + "keep_local": True, + } + }, + ) + config = _config_with_local_root(local_root) + resp = browse.scan_folder_sync(str(run_dir), config) + by_name = {e.name: e for e in resp.entries} + assert by_name["scan.tif"].sync_status == "synced" + assert by_name["scan.tif"].keep_local is True + + +def test_build_run_node_rollup_from_sync_state(tmp_path: Path) -> None: + """The tree run-node sync_status is the sync_state.json rollup, not creation.json.""" + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + _write_sync_state( + run_dir, + {"scan.tif": {"synced_signature": (10, 1), "verified_at": "2026-05-21T01:00:00Z"}}, + ) + deps = AppDependencies(config=_config_with_local_root(local_root)) + app = create_app(dependencies=deps) + client = TestClient(app) + body = client.get("/api/v1/tree").json() + run = body["equipment"][0]["projects"][0]["runs"][0] + assert run["sync_status"] == RunSyncState.SYNCED.value + + +def test_build_run_node_rollup_cleared(tmp_path: Path) -> None: + """A cleared run rolls up to ``cleared`` in the tree.""" + local_root = tmp_path / "data" + run_dir = _seed_run_with_creation(local_root) + _write_sync_state( + run_dir, + {"scan.tif": {"synced_signature": (10, 1), "verified_at": "2026-05-21T01:00:00Z"}}, + cleared=True, + ) + deps = AppDependencies(config=_config_with_local_root(local_root)) + app = create_app(dependencies=deps) + client = TestClient(app) + body = client.get("/api/v1/tree").json() + run = body["equipment"][0]["projects"][0]["runs"][0] + assert run["sync_status"] == RunSyncState.CLEARED.value diff --git a/tests/unit/api/test_config_router.py b/tests/unit/api/test_config_router.py index 9e92fb5..84b99c2 100644 --- a/tests/unit/api/test_config_router.py +++ b/tests/unit/api/test_config_router.py @@ -29,8 +29,6 @@ def _ready_config() -> Config: label="Equipment 1", local_root="/d", nas_root="/n", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -117,8 +115,6 @@ async def saver(config: Config) -> None: "label": "Flow Cytometer 99", "local_root": "/data", "nas_root": "/srv/nas", - "completeness_signal": "sentinel_file", - "sentinel_filename": "done.flag", "transport": { "type": "rclone", "rclone_remote": "lab-nas", @@ -144,8 +140,6 @@ def test_append_equipment_rejects_duplicate_id() -> None: "label": "Equipment 1 duplicate", "local_root": "/data", "nas_root": "/srv/nas", - "completeness_signal": "sentinel_file", - "sentinel_filename": "done.flag", "transport": { "type": "rclone", "rclone_remote": "lab-nas", diff --git a/tests/unit/api/test_health.py b/tests/unit/api/test_health.py index 8405183..6b6ac3d 100644 --- a/tests/unit/api/test_health.py +++ b/tests/unit/api/test_health.py @@ -27,8 +27,6 @@ def _ready_config() -> Config: label="Equipment 1", local_root="/d", nas_root="/n", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/unit/api/test_operations.py b/tests/unit/api/test_operations.py index 3e73d38..0221484 100644 --- a/tests/unit/api/test_operations.py +++ b/tests/unit/api/test_operations.py @@ -29,8 +29,6 @@ def _ready_config() -> Config: label="Equipment 1", local_root="/d", nas_root="/n", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/unit/api/test_problems.py b/tests/unit/api/test_problems.py index 652606e..3a7142e 100644 --- a/tests/unit/api/test_problems.py +++ b/tests/unit/api/test_problems.py @@ -61,8 +61,6 @@ def _ready_config(local_root: Path) -> Config: label="Equipment 1", local_root=str(local_root), nas_root="/n", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/unit/api/test_schemas.py b/tests/unit/api/test_schemas.py index 7a29dc3..1fb824a 100644 --- a/tests/unit/api/test_schemas.py +++ b/tests/unit/api/test_schemas.py @@ -17,7 +17,6 @@ from exlab_wizard.api.schemas import ( CreationJson, EquipmentJson, - IngestJson, LimsProjectBlock, OrchestratorBlock, OverrideEntry, @@ -39,9 +38,7 @@ from exlab_wizard.api.schemas import TestRunsJson as RunsTestMarkerJson from exlab_wizard.constants import ( CreationLevel, - IngestState, LIMSProjectSource, - OrchestratorTransportType, PluginStatus, RunKind, RunScope, @@ -151,31 +148,27 @@ def test_creation_json_orchestrator_block_omitted_when_none() -> None: assert '"orchestrator"' not in encoded -def test_orchestrator_block_carries_relay_discovery_fields() -> None: - """Redesign §3.3: pushed creation.json carries the equipment label + - completeness signal so the orchestrator can auto-discover received - equipment without a per-equipment registry of its own.""" +def test_orchestrator_block_carries_relay_discovery_field() -> None: + """Redesign §3.3: pushed creation.json carries the equipment label so + the orchestrator can auto-discover received equipment without a + per-equipment registry of its own. The operator-free quiescence-sync + redesign (2026-05-21) removed the completeness-signal relay fields.""" payload = _minimal_creation_json( orchestrator=OrchestratorBlock( enabled=True, host="labpc-04", label="Lab Acquisition Station 01", equipment_label="Confocal Microscope 1", - completeness_signal="sentinel_file", - sentinel_filename="acquisition_complete.flag", ), ) encoded = msgspec_json.encode(payload) decoded = msgspec_json.decode(encoded, type=CreationJson) assert decoded.orchestrator is not None assert decoded.orchestrator.equipment_label == "Confocal Microscope 1" - assert decoded.orchestrator.completeness_signal == "sentinel_file" - assert decoded.orchestrator.sentinel_filename == "acquisition_complete.flag" - assert decoded.orchestrator.manifest_filename is None -def test_orchestrator_block_relay_fields_default_to_empty_or_none() -> None: - """Older creation.json files (no relay fields) decode cleanly.""" +def test_orchestrator_block_relay_field_defaults_to_none() -> None: + """Older creation.json files (no relay field) decode cleanly.""" payload = _minimal_creation_json( orchestrator=OrchestratorBlock( enabled=True, host="labpc-04", label="Lab Acquisition Station 01" @@ -185,9 +178,6 @@ def test_orchestrator_block_relay_fields_default_to_empty_or_none() -> None: decoded = msgspec_json.decode(encoded, type=CreationJson) assert decoded.orchestrator is not None assert decoded.orchestrator.equipment_label is None - assert decoded.orchestrator.completeness_signal is None - assert decoded.orchestrator.sentinel_filename is None - assert decoded.orchestrator.manifest_filename is None def test_creation_json_default_sync_status_is_pending() -> None: @@ -485,22 +475,6 @@ def test_test_runs_json_missing_required_field_raises() -> None: # --------------------------------------------------------------------------- -def _ingest_json_payload(**overrides: object) -> bytes: - """Build a minimal valid ingest.json JSON document for round-trip tests.""" - base: dict[str, object] = { - "schema_version": "1.1", - "project_name": "PROJ-0042", - "equipment_id": "CONFOCAL_01", - "run_kind": "experimental", - "run_path": "/staging/Run_2026-04-17", - "transport": "smb_mount", - "current_state": "staging", - "history": [], - } - base.update(overrides) - return msgspec_json.encode(base) - - def _creation_json_payload(**overrides: object) -> bytes: """Build a minimal valid creation.json JSON document for round-trip tests.""" base: dict[str, object] = { @@ -577,72 +551,6 @@ def _creation_json_payload(**overrides: object) -> bytes: SyncStatus.BLOCKED_BY_VALIDATION, lambda: _creation_json_payload(sync_status="blocked_by_validation"), ), - # IngestJson.run_kind - ( - IngestJson, - "run_kind", - "experimental", - RunKind.EXPERIMENTAL, - lambda: _ingest_json_payload(run_kind="experimental"), - ), - ( - IngestJson, - "run_kind", - "test", - RunKind.TEST, - lambda: _ingest_json_payload(run_kind="test"), - ), - # IngestJson.transport - ( - IngestJson, - "transport", - "smb_mount", - OrchestratorTransportType.SMB_MOUNT, - lambda: _ingest_json_payload(transport="smb_mount"), - ), - ( - IngestJson, - "transport", - "file_transfer", - OrchestratorTransportType.FILE_TRANSFER, - lambda: _ingest_json_payload(transport="file_transfer"), - ), - # IngestJson.current_state - ( - IngestJson, - "current_state", - "staging", - IngestState.STAGING, - lambda: _ingest_json_payload(current_state="staging"), - ), - ( - IngestJson, - "current_state", - "complete", - IngestState.COMPLETE, - lambda: _ingest_json_payload(current_state="complete"), - ), - ( - IngestJson, - "current_state", - "sync_queued", - IngestState.SYNC_QUEUED, - lambda: _ingest_json_payload(current_state="sync_queued"), - ), - ( - IngestJson, - "current_state", - "sync_verified", - IngestState.SYNC_VERIFIED, - lambda: _ingest_json_payload(current_state="sync_verified"), - ), - ( - IngestJson, - "current_state", - "cleared", - IngestState.CLEARED, - lambda: _ingest_json_payload(current_state="cleared"), - ), ], ) def test_strenum_field_round_trip_preserves_wire_format( diff --git a/tests/unit/api/test_sessions.py b/tests/unit/api/test_sessions.py index 5c21347..4816a09 100644 --- a/tests/unit/api/test_sessions.py +++ b/tests/unit/api/test_sessions.py @@ -37,8 +37,6 @@ def _ready_config() -> Config: label="Equipment 1", local_root="/d", nas_root="/n", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/unit/api/test_setup.py b/tests/unit/api/test_setup.py index 7a32946..97a7106 100644 --- a/tests/unit/api/test_setup.py +++ b/tests/unit/api/test_setup.py @@ -40,8 +40,6 @@ def _ready_config() -> Config: label="Equipment 1", local_root="/data", nas_root="/srv/nas", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", diff --git a/tests/unit/api/test_staging_router.py b/tests/unit/api/test_staging_router.py index a8c2cf6..d3433c4 100644 --- a/tests/unit/api/test_staging_router.py +++ b/tests/unit/api/test_staging_router.py @@ -1,16 +1,21 @@ -"""Unit tests for the ``/staging`` router. Backend Spec §13.7, §13.8.""" +"""Unit tests for the ``/staging`` router. Backend Spec §13.7, §13.8. + +The operator-free per-file NAS sync redesign (2026-05-21) removed +``ingest.json``; Phase 5 derives a run's ``current_state`` from the +``sync_state.json`` rollup (``syncing`` / ``synced`` / ``cleared``), and +the clearable set is the fully-``synced`` rollup. +""" from __future__ import annotations +import asyncio from dataclasses import dataclass from pathlib import Path -import msgspec from fastapi.testclient import TestClient from exlab_wizard.api import AppDependencies, create_app -from exlab_wizard.api.schemas import IngestJson -from exlab_wizard.cache.ingest_writer import IngestWriter +from exlab_wizard.cache.sync_state_writer import SyncStateWriter from exlab_wizard.config.models import ( Config, EquipmentConfig, @@ -19,12 +24,7 @@ PathsConfig, RcloneTransport, ) -from exlab_wizard.constants import ( - CACHE_DIR_NAME, - INGEST_JSON_NAME, - INGEST_JSON_VERSION, - IngestState, -) +from exlab_wizard.constants import CACHE_DIR_NAME, RUNS_DIR_NAME # --------------------------------------------------------------------------- # Helpers @@ -39,6 +39,8 @@ class _Handle: class _StubNasSync: + """In-memory sync-queue stub recording ``enqueue`` calls (force-sync).""" + def __init__(self) -> None: self.enqueued: list[Path] = [] @@ -56,8 +58,6 @@ def _make_config(staging_root: Path, *, enabled: bool = True) -> Config: label="Equipment 1", local_root=str(staging_root), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="done.flag", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -73,42 +73,85 @@ def _make_config(staging_root: Path, *, enabled: bool = True) -> Config: ) -async def _seed_run( +def _seed_run( staging_root: Path, *, - state: IngestState = IngestState.STAGING, - last_at: str = "2026-04-17T12:00:00Z", + equipment: str = "EQ1", + project: str = "PROJ-0001", + run_name: str = "Run_2026-04-17T14-32-00", ) -> Path: - run_dir = staging_root / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" + run_dir = staging_root / equipment / project / RUNS_DIR_NAME / run_name run_dir.mkdir(parents=True) (run_dir / "data.bin").write_bytes(b"x" * 100) + return run_dir + + +def _write_creation_json(run_dir: Path) -> None: + """Write a minimal ``creation.json`` so a dir reads as a real run. + + The ``POST /staging/{run}/keep-local`` endpoint guards on the + presence of this cache (operator-free per-file NAS sync design, + 2026-05-21) before letting ``SyncStateWriter`` create ``sync_state.json``. + """ + import msgspec + + from exlab_wizard.api.schemas import ( + CreationJson, + LimsProjectBlock, + PathsBlock, + TemplateBlock, + ) + from exlab_wizard.constants import CREATION_JSON_NAME, CREATION_JSON_VERSION + cache = run_dir / CACHE_DIR_NAME - cache.mkdir() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "PROJ-0001", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": str(run_dir), - "transport": "smb_mount", - "current_state": state.value, - "history": [{"state": state.value, "at": last_at, "host": "host"}], - }, - type=IngestJson, + cache.mkdir(parents=True, exist_ok=True) + payload = CreationJson( + schema_version=CREATION_JSON_VERSION, + created_at="2026-05-21T00:00:00Z", + created_by="asmith", + level="run", + run_kind="experimental", + lims_project=LimsProjectBlock( + uid="x", short_id="PROJ-0001", name_at_creation="example", source="live" + ), + template=TemplateBlock( + name="basic", version="1.0.0", source_path="/tpl/basic", run_scope="experimental" + ), + variables={}, + paths=PathsBlock(local=str(run_dir), nas="/srv/nas/EQ1"), ) - await IngestWriter().write_ingest(cache / INGEST_JSON_NAME, payload) + (cache / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(payload)) + + +def _seed_real_run( + staging_root: Path, + *, + equipment: str = "EQ1", + project: str = "PROJ-0001", + run_name: str = "Run_2026-04-17T14-32-00", +) -> Path: + """Seed a run dir that carries a ``creation.json`` -- a real run.""" + run_dir = _seed_run(staging_root, equipment=equipment, project=project, run_name=run_name) + _write_creation_json(run_dir) return run_dir +def _mark_synced(run_dir: Path) -> None: + """Write a fully-verified ``sync_state.json`` so the run rolls up to ``synced``.""" + writer = SyncStateWriter() + asyncio.run( + writer.upsert_file( + run_dir, "data.bin", synced_signature=(100, 1), verified_at="2026-05-21T00:00:00Z" + ) + ) + + # --------------------------------------------------------------------------- -# 503 when orchestrator disabled +# No 503 gate (Redesign §3.1: orchestrator pipeline always on) # --------------------------------------------------------------------------- def test_get_staging_returns_empty_when_staging_root_unset(tmp_path: Path) -> None: - """Redesign §3.1: orchestrator pipeline is always on. The 503 gate is - removed; a staging_root that isn't on disk returns an empty list.""" config = _make_config(tmp_path, enabled=False) deps = AppDependencies(config=config) app = create_app(dependencies=deps) @@ -118,25 +161,21 @@ def test_get_staging_returns_empty_when_staging_root_unset(tmp_path: Path) -> No assert resp.json() == {"runs": []} -def test_force_sync_returns_404_when_run_missing(tmp_path: Path) -> None: +def test_force_sync_returns_no_503_when_run_missing(tmp_path: Path) -> None: config = _make_config(tmp_path, enabled=False) deps = AppDependencies(config=config, nas_sync=_StubNasSync()) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.post("/api/v1/staging/some/path/force-sync") - # Either 404 (run not found) or 200 (stub queues it); the spec doesn't - # mandate which, only that there's no orchestrator_disabled gate. assert resp.status_code != 503 -def test_clear_returns_404_when_run_missing(tmp_path: Path) -> None: +def test_clear_returns_no_503_when_run_missing(tmp_path: Path) -> None: config = _make_config(tmp_path, enabled=False) deps = AppDependencies(config=config) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.post("/api/v1/staging/some/path/clear") - # Redesign §3.1: no orchestrator_disabled gate; the endpoint just - # reports a 404 / 200 outcome depending on whether the run exists. assert resp.status_code != 503 @@ -145,10 +184,10 @@ def test_clear_returns_404_when_run_missing(tmp_path: Path) -> None: # --------------------------------------------------------------------------- -async def test_get_staging_returns_run_rows(tmp_path: Path) -> None: - await _seed_run(tmp_path) +def test_get_staging_returns_run_rows(tmp_path: Path) -> None: + _seed_run(tmp_path) config = _make_config(tmp_path) - deps = AppDependencies(config=config) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.get("/api/v1/staging") @@ -158,14 +197,25 @@ async def test_get_staging_returns_run_rows(tmp_path: Path) -> None: assert len(body["runs"]) == 1 row = body["runs"][0] assert row["equipment_id"] == "EQ1" - assert row["current_state"] == IngestState.STAGING.value + # No sync_state.json yet -> rolls up to "syncing". + assert row["current_state"] == "syncing" assert row["run_kind"] == "experimental" assert row["file_count"] == 1 assert row["byte_total"] == 100 +def test_get_staging_derives_current_state_from_sync_state_rollup(tmp_path: Path) -> None: + run_dir = _seed_run(tmp_path) + _mark_synced(run_dir) + config = _make_config(tmp_path) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.get("/api/v1/staging") + assert resp.json()["runs"][0]["current_state"] == "synced" + + def test_get_staging_returns_empty_runs_for_missing_root(tmp_path: Path) -> None: - """A staging_root that doesn't exist on disk yet returns runs=[].""" config = _make_config(tmp_path / "missing") deps = AppDependencies(config=config) app = create_app(dependencies=deps) @@ -180,8 +230,8 @@ def test_get_staging_returns_empty_runs_for_missing_root(tmp_path: Path) -> None # --------------------------------------------------------------------------- -async def test_force_sync_invokes_nas_sync_enqueue(tmp_path: Path) -> None: - run_dir = await _seed_run(tmp_path) +def test_force_sync_invokes_nas_sync_enqueue(tmp_path: Path) -> None: + run_dir = _seed_run(tmp_path) nas_sync = _StubNasSync() config = _make_config(tmp_path) deps = AppDependencies(config=config, nas_sync=nas_sync) @@ -209,10 +259,11 @@ def test_force_sync_returns_503_when_nas_sync_unwired(tmp_path: Path) -> None: # --------------------------------------------------------------------------- -async def test_clear_endpoint_deletes_sync_verified_run(tmp_path: Path) -> None: - run_dir = await _seed_run(tmp_path, state=IngestState.SYNC_VERIFIED) +def test_clear_endpoint_deletes_synced_run(tmp_path: Path) -> None: + run_dir = _seed_run(tmp_path) + _mark_synced(run_dir) config = _make_config(tmp_path) - deps = AppDependencies(config=config, ingest_writer=IngestWriter()) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.post(f"/api/v1/staging/{run_dir}/clear") @@ -220,13 +271,19 @@ async def test_clear_endpoint_deletes_sync_verified_run(tmp_path: Path) -> None: body = resp.json() assert body["files_freed"] >= 1 assert body["bytes_freed"] >= 100 - assert not run_dir.exists() + # Data file is gone; the .exlab-wizard/ metadata subtree survives so the + # cleared run still renders "On NAS" tombstones. + assert not (run_dir / "data.bin").exists() + assert (run_dir / CACHE_DIR_NAME).exists() + # ``cleared_at`` was stamped -> rollup is now CLEARED. + state = asyncio.run(SyncStateWriter().read(run_dir)) + assert state.cleared_at is not None -async def test_clear_endpoint_rejects_non_sync_verified_run(tmp_path: Path) -> None: - run_dir = await _seed_run(tmp_path, state=IngestState.STAGING) +def test_clear_endpoint_rejects_non_synced_run(tmp_path: Path) -> None: + run_dir = _seed_run(tmp_path) # no sync_state.json -> rollup "syncing" config = _make_config(tmp_path) - deps = AppDependencies(config=config, ingest_writer=IngestWriter()) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.post(f"/api/v1/staging/{run_dir}/clear") @@ -235,13 +292,12 @@ async def test_clear_endpoint_rejects_non_sync_verified_run(tmp_path: Path) -> N assert run_dir.exists() -def test_clear_endpoint_falls_back_to_default_writer_when_unwired(tmp_path: Path) -> None: - """A deps without ``ingest_writer`` builds a fresh IngestWriter.""" +def test_clear_endpoint_idempotent_for_missing_run(tmp_path: Path) -> None: + """A run with no queue job and no directory clears to zeros.""" config = _make_config(tmp_path) - deps = AppDependencies(config=config) # no ingest_writer + deps = AppDependencies(config=config) app = create_app(dependencies=deps) with TestClient(app) as client: - # Path doesn't exist -- clear is idempotent and returns zeros. resp = client.post("/api/v1/staging/missing/path/clear") assert resp.status_code == 200 body = resp.json() @@ -254,85 +310,36 @@ def test_clear_endpoint_falls_back_to_default_writer_when_unwired(tmp_path: Path # --------------------------------------------------------------------------- -async def test_clear_verified_endpoint_clears_only_sync_verified_runs( - tmp_path: Path, -) -> None: - """The bulk endpoint clears every sync_verified run and reports paths.""" - # Two SYNC_VERIFIED runs + one STAGING run that must remain. - verified_a = await _seed_run(tmp_path, state=IngestState.SYNC_VERIFIED) - # Second verified run under a separate project so its directory is distinct. - second_dir = tmp_path / "EQ1" / "PROJ-0002" / "Run_2026-05-05" - second_dir.mkdir(parents=True) - (second_dir / "file.bin").write_bytes(b"y" * 50) - cache = second_dir / CACHE_DIR_NAME - cache.mkdir() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "PROJ-0002", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": str(second_dir), - "transport": "smb_mount", - "current_state": IngestState.SYNC_VERIFIED.value, - "history": [ - { - "state": IngestState.SYNC_VERIFIED.value, - "at": "2026-05-05T10:00:00Z", - "host": "h", - } - ], - }, - type=IngestJson, - ) - await IngestWriter().write_ingest(cache / INGEST_JSON_NAME, payload) - staging_dir = tmp_path / "EQ2" / "PROJ-0003" / "Run_2026-05-06" - staging_dir.mkdir(parents=True) - (staging_dir / "raw.bin").write_bytes(b"z" * 25) - staging_cache = staging_dir / CACHE_DIR_NAME - staging_cache.mkdir() - staging_payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "PROJ-0003", - "equipment_id": "EQ2", - "run_kind": "experimental", - "run_path": str(staging_dir), - "transport": "smb_mount", - "current_state": IngestState.STAGING.value, - "history": [ - { - "state": IngestState.STAGING.value, - "at": "2026-05-06T10:00:00Z", - "host": "h", - } - ], - }, - type=IngestJson, - ) - await IngestWriter().write_ingest(staging_cache / INGEST_JSON_NAME, staging_payload) +def test_clear_verified_endpoint_clears_only_synced_runs(tmp_path: Path) -> None: + """The bulk endpoint clears every fully-``synced`` run and reports paths.""" + synced_a = _seed_run(tmp_path) + synced_b = _seed_run(tmp_path, project="PROJ-0002", run_name="Run_2026-05-05") + syncing_run = _seed_run(tmp_path, equipment="EQ2", project="PROJ-0003") + _mark_synced(synced_a) + _mark_synced(synced_b) + # ``syncing_run`` has no sync_state.json -> rollup "syncing". config = _make_config(tmp_path) - deps = AppDependencies(config=config, ingest_writer=IngestWriter()) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.post("/api/v1/staging/clear-verified") assert resp.status_code == 200 body = resp.json() - assert set(body["cleared_paths"]) == {str(verified_a), str(second_dir)} - assert not verified_a.exists() - assert not second_dir.exists() - # The STAGING run must NOT be touched by the bulk action. - assert staging_dir.exists() + assert set(body["cleared_paths"]) == {str(synced_a), str(synced_b)} + # Data files are gone; the metadata subtree survives for tombstones. + assert not (synced_a / "data.bin").exists() + assert not (synced_b / "data.bin").exists() + assert (synced_a / CACHE_DIR_NAME).exists() + # The still-syncing run's data must NOT be touched by the bulk action. + assert (syncing_run / "data.bin").exists() -def test_clear_verified_endpoint_returns_empty_when_no_verified_runs( - tmp_path: Path, -) -> None: - """No SYNC_VERIFIED rows -> empty cleared_paths, no error.""" +def test_clear_verified_endpoint_returns_empty_when_no_verified_runs(tmp_path: Path) -> None: + """No verified rows -> empty cleared_paths, no error.""" config = _make_config(tmp_path) # empty staging_root - deps = AppDependencies(config=config, ingest_writer=IngestWriter()) + deps = AppDependencies(config=config) app = create_app(dependencies=deps) with TestClient(app) as client: resp = client.post("/api/v1/staging/clear-verified") @@ -348,3 +355,146 @@ def test_clear_verified_endpoint_returns_503_when_config_unwired(tmp_path: Path) with TestClient(app) as client: resp = client.post("/api/v1/staging/clear-verified") assert resp.status_code == 503 + + +# --------------------------------------------------------------------------- +# POST /staging/{run_path}/keep-local +# --------------------------------------------------------------------------- + + +def test_keep_local_toggles_sync_state(tmp_path: Path) -> None: + """The endpoint flips ``keep_local`` in the run's ``sync_state.json``.""" + run_dir = _seed_real_run(tmp_path) + writer = SyncStateWriter() + config = _make_config(tmp_path) + deps = AppDependencies(config=config, sync_state_writer=writer) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + f"/api/v1/staging/{run_dir}/keep-local", + json={"relative_path": "data.bin", "keep_local": True}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body == { + "run_path": str(run_dir), + "relative_path": "data.bin", + "keep_local": True, + } + state = writer.read_sync(run_dir) + assert state.files["data.bin"].keep_local is True + + +def test_keep_local_toggle_off_round_trips(tmp_path: Path) -> None: + """Setting ``keep_local`` False after True clears the flag on disk.""" + run_dir = _seed_real_run(tmp_path) + writer = SyncStateWriter() + asyncio.run(writer.set_keep_local(run_dir, "data.bin", True)) + config = _make_config(tmp_path) + deps = AppDependencies(config=config, sync_state_writer=writer) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + f"/api/v1/staging/{run_dir}/keep-local", + json={"relative_path": "data.bin", "keep_local": False}, + ) + assert resp.status_code == 200 + assert resp.json()["keep_local"] is False + assert writer.read_sync(run_dir).files["data.bin"].keep_local is False + + +def test_keep_local_returns_503_when_writer_unwired(tmp_path: Path) -> None: + """A deps without a ``sync_state_writer`` raises the standard 503.""" + run_dir = _seed_real_run(tmp_path) + config = _make_config(tmp_path) + deps = AppDependencies(config=config) # no sync_state_writer + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + f"/api/v1/staging/{run_dir}/keep-local", + json={"relative_path": "data.bin", "keep_local": True}, + ) + assert resp.status_code == 503 + + +def test_keep_local_returns_503_when_config_unwired(tmp_path: Path) -> None: + """A deps without a config raises the standard 503.""" + deps = AppDependencies(config=None) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + "/api/v1/staging/some/run/keep-local", + json={"relative_path": "data.bin", "keep_local": True}, + ) + assert resp.status_code == 503 + + +def test_keep_local_rejects_extra_body_fields(tmp_path: Path) -> None: + """The request model forbids unknown fields.""" + run_dir = _seed_real_run(tmp_path) + config = _make_config(tmp_path) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + f"/api/v1/staging/{run_dir}/keep-local", + json={"relative_path": "data.bin", "keep_local": True, "bogus": 1}, + ) + assert resp.status_code == 422 + + +def test_keep_local_returns_404_for_path_outside_allowed_roots(tmp_path: Path) -> None: + """A path outside the configured roots is rejected before any write. + + Operator-free per-file NAS sync design (2026-05-21): ``run_path`` + comes straight from the URL and ``SyncStateWriter`` *creates* the + run's ``sync_state.json``. A hostile path (``/etc``) must 404, not + provoke a write under a system directory. + """ + config = _make_config(tmp_path) + deps = AppDependencies(config=config, sync_state_writer=SyncStateWriter()) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + "/api/v1/staging//etc/keep-local", + json={"relative_path": "passwd", "keep_local": True}, + ) + assert resp.status_code == 404 + # The guard must not have created sync_state.json under /etc. + assert not (Path("/etc") / CACHE_DIR_NAME / "sync_state.json").exists() + + +def test_keep_local_returns_404_when_run_has_no_creation_json(tmp_path: Path) -> None: + """A dir under an allowed root but without a creation.json is not a run.""" + # Inside local_root but no creation.json -> not a real run. + run_dir = _seed_run(tmp_path) # _seed_run omits creation.json + config = _make_config(tmp_path) + writer = SyncStateWriter() + deps = AppDependencies(config=config, sync_state_writer=writer) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + f"/api/v1/staging/{run_dir}/keep-local", + json={"relative_path": "data.bin", "keep_local": True}, + ) + assert resp.status_code == 404 + # No sync_state.json was created for the non-run directory. + assert not (run_dir / CACHE_DIR_NAME / "sync_state.json").exists() + + +def test_keep_local_creates_sync_state_when_cache_dir_absent(tmp_path: Path) -> None: + """The writer mkdir's the .exlab-wizard/ dir -- a real run with only a + creation.json (cache dir exists) toggles cleanly, and even a run whose + cache dir is later removed does not 500 (S2 robustness).""" + run_dir = _seed_real_run(tmp_path) + config = _make_config(tmp_path) + writer = SyncStateWriter() + deps = AppDependencies(config=config, sync_state_writer=writer) + app = create_app(dependencies=deps) + with TestClient(app) as client: + resp = client.post( + f"/api/v1/staging/{run_dir}/keep-local", + json={"relative_path": "data.bin", "keep_local": True}, + ) + assert resp.status_code == 200 + assert writer.read_sync(run_dir).files["data.bin"].keep_local is True diff --git a/tests/unit/cache/test_ingest_writer.py b/tests/unit/cache/test_ingest_writer.py deleted file mode 100644 index cec41cd..0000000 --- a/tests/unit/cache/test_ingest_writer.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Unit tests for ``exlab_wizard.cache.ingest_writer``. - -Covers Backend Spec §13.3 (state machine), §13.4 (on-disk shape), and -§4.4.5 (atomic + locked write contract). Each transition path is verified -end-to-end against an actual file on ``tmp_path`` so the atomic-replace -codepath is exercised. - -Constructing an ``IngestJson`` requires Agent B's -``exlab_wizard.api.schemas`` -- if those Structs are not yet present these -tests fail at import time, which is the expected behaviour until the -parallel agents are integrated. -""" - -from __future__ import annotations - -import asyncio -from pathlib import Path - -import msgspec -import pytest - -from exlab_wizard.api.schemas import IngestJson -from exlab_wizard.cache.ingest_writer import IngestWriter -from exlab_wizard.constants import INGEST_JSON_NAME, INGEST_JSON_VERSION, IngestState - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -_HOST = "labpc-04" - - -def _make_payload(**overrides: object) -> IngestJson: - """Construct a minimal valid ``IngestJson`` for testing. - - The starting payload sits in the ``staging`` state with an empty - ``history`` list. Tests that need a specific shape can override any - field via ``**overrides``. - """ - base: dict[str, object] = { - "schema_version": INGEST_JSON_VERSION, - "project_name": "Cortex Q3 Pilot", - "equipment_id": "CONFOCAL_01", - "run_kind": "experimental", - "run_path": "CONFOCAL_01/PROJ-0042/Run_2026-04-17T14-32-00", - "transport": "smb_mount", - "current_state": IngestState.STAGING.value, - "history": [ - { - "state": IngestState.STAGING.value, - "at": "2026-04-17T14:35:00Z", - "host": _HOST, - }, - ], - } - base.update(overrides) - return msgspec.convert(base, type=IngestJson) - - -def _ingest_path(tmp_path: Path) -> Path: - return tmp_path / ".exlab-wizard" / INGEST_JSON_NAME - - -# --------------------------------------------------------------------------- -# write_ingest / read_ingest -# --------------------------------------------------------------------------- - - -async def test_write_ingest_creates_a_valid_v1_1_file(tmp_path: Path) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - payload = _make_payload() - - await writer.write_ingest(path, payload) - - assert path.exists() - raw: dict = msgspec.json.decode(path.read_bytes()) - assert raw["schema_version"] == INGEST_JSON_VERSION - assert raw["schema_version"].startswith("1.") - assert raw["current_state"] == IngestState.STAGING.value - assert raw["project_name"] == "Cortex Q3 Pilot" - assert raw["equipment_id"] == "CONFOCAL_01" - assert raw["run_kind"] == "experimental" - assert raw["transport"] == "smb_mount" - assert isinstance(raw["history"], list) - - -async def test_write_ingest_creates_parent_directories(tmp_path: Path) -> None: - writer = IngestWriter() - nested = tmp_path / "a" / "b" / ".exlab-wizard" / INGEST_JSON_NAME - payload = _make_payload() - - await writer.write_ingest(nested, payload) - - assert nested.exists() - - -async def test_read_ingest_round_trips(tmp_path: Path) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - payload = _make_payload() - await writer.write_ingest(path, payload) - - roundtrip = await writer.read_ingest(path) - - assert roundtrip.schema_version == payload.schema_version - assert roundtrip.project_name == payload.project_name - assert roundtrip.equipment_id == payload.equipment_id - assert roundtrip.run_kind == payload.run_kind - assert roundtrip.run_path == payload.run_path - assert roundtrip.transport == payload.transport - assert roundtrip.current_state == payload.current_state - assert roundtrip.history == payload.history - - -# --------------------------------------------------------------------------- -# append_state_transition -- valid forward transitions -# --------------------------------------------------------------------------- - - -async def test_append_state_transition_updates_current_state_and_history( - tmp_path: Path, -) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - await writer.write_ingest(path, _make_payload()) - - new_payload = await writer.append_state_transition( - path, - IngestState.COMPLETE, - host=_HOST, - files_received=142, - bytes_received=48_293_847_234, - ) - - assert new_payload.current_state == IngestState.COMPLETE.value - assert len(new_payload.history) == 2 - last = new_payload.history[-1] - assert last["state"] == IngestState.COMPLETE.value - assert last["host"] == _HOST - assert "at" in last - # The new entry must be persisted to disk, not just held in memory. - on_disk = await writer.read_ingest(path) - assert on_disk.current_state == new_payload.current_state - assert on_disk.history == new_payload.history - - -async def test_append_complete_records_files_and_bytes(tmp_path: Path) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - await writer.write_ingest(path, _make_payload()) - - new_payload = await writer.append_state_transition( - path, - IngestState.COMPLETE, - host=_HOST, - files_received=142, - bytes_received=48_293_847_234, - ) - - last = new_payload.history[-1] - assert last["files_received"] == 142 - assert last["bytes_received"] == 48_293_847_234 - - -async def test_append_sync_verified_records_nas_path_and_checksum( - tmp_path: Path, -) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - # Walk the state machine forward to sync_queued before transitioning to - # sync_verified -- the writer rejects non-forward transitions. - payload = _make_payload( - current_state=IngestState.SYNC_QUEUED.value, - history=[ - {"state": IngestState.STAGING.value, "at": "t0", "host": _HOST}, - {"state": IngestState.COMPLETE.value, "at": "t1", "host": _HOST}, - {"state": IngestState.SYNC_QUEUED.value, "at": "t2", "host": _HOST}, - ], - ) - await writer.write_ingest(path, payload) - - new_payload = await writer.append_state_transition( - path, - IngestState.SYNC_VERIFIED, - host=_HOST, - nas_path="//nas01/lab/CONFOCAL_01/PROJ-0042/Run_2026-04-17T14-32-00", - checksum_file=".exlab-wizard/checksums.sha256", - ) - - last = new_payload.history[-1] - assert last["state"] == IngestState.SYNC_VERIFIED.value - assert last["nas_path"] == "//nas01/lab/CONFOCAL_01/PROJ-0042/Run_2026-04-17T14-32-00" - assert last["checksum_file"] == ".exlab-wizard/checksums.sha256" - - -async def test_append_intermediate_states_do_not_carry_complete_extras( - tmp_path: Path, -) -> None: - """``files_received`` is only meaningful for the ``complete`` transition. - - The writer drops the extras silently for non-matching states so callers - don't accidentally inject misleading audit entries. - """ - writer = IngestWriter() - path = _ingest_path(tmp_path) - payload = _make_payload( - current_state=IngestState.COMPLETE.value, - history=[ - {"state": IngestState.STAGING.value, "at": "t0", "host": _HOST}, - {"state": IngestState.COMPLETE.value, "at": "t1", "host": _HOST}, - ], - ) - await writer.write_ingest(path, payload) - - new_payload = await writer.append_state_transition( - path, - IngestState.SYNC_QUEUED, - host=_HOST, - files_received=99, # non-matching extras must be dropped - bytes_received=99, - ) - last = new_payload.history[-1] - assert "files_received" not in last - assert "bytes_received" not in last - - -# --------------------------------------------------------------------------- -# append_state_transition -- backward / illegal transitions raise ValueError -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - ("start_state", "bad_target"), - [ - # cleared -> staging is the canonical "going backward" example from - # the spec text in §13.3. - (IngestState.CLEARED, IngestState.STAGING), - (IngestState.SYNC_VERIFIED, IngestState.STAGING), - (IngestState.COMPLETE, IngestState.STAGING), - # Skipping a state forward is also illegal -- only single-step - # forward transitions from §13.3 are permitted. - (IngestState.STAGING, IngestState.SYNC_QUEUED), - (IngestState.STAGING, IngestState.SYNC_VERIFIED), - (IngestState.STAGING, IngestState.CLEARED), - (IngestState.COMPLETE, IngestState.SYNC_VERIFIED), - (IngestState.SYNC_QUEUED, IngestState.CLEARED), - # Self-loops are rejected (no-op transitions are not allowed). - (IngestState.STAGING, IngestState.STAGING), - (IngestState.COMPLETE, IngestState.COMPLETE), - ], -) -async def test_append_state_transition_rejects_illegal_transitions( - tmp_path: Path, - start_state: IngestState, - bad_target: IngestState, -) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - payload = _make_payload( - current_state=start_state.value, - history=[{"state": start_state.value, "at": "t0", "host": _HOST}], - ) - await writer.write_ingest(path, payload) - - with pytest.raises( - ValueError, - match=r"illegal state transition|Invalid ingest state transition", - ): - await writer.append_state_transition(path, bad_target, host=_HOST) - - -async def test_append_state_transition_does_not_mutate_file_on_failure( - tmp_path: Path, -) -> None: - writer = IngestWriter() - path = _ingest_path(tmp_path) - await writer.write_ingest(path, _make_payload()) - before = path.read_bytes() - - with pytest.raises(ValueError): - await writer.append_state_transition( - path, - IngestState.SYNC_VERIFIED, # not reachable from staging in one hop - host=_HOST, - ) - - assert path.read_bytes() == before - - -# --------------------------------------------------------------------------- -# History preservation across multiple sequential transitions -# --------------------------------------------------------------------------- - - -async def test_history_is_preserved_across_five_sequential_transitions( - tmp_path: Path, -) -> None: - """Walk the entire state machine and check no entry is dropped.""" - writer = IngestWriter() - path = _ingest_path(tmp_path) - await writer.write_ingest(path, _make_payload()) - - # staging -> complete - await writer.append_state_transition( - path, - IngestState.COMPLETE, - host=_HOST, - files_received=10, - bytes_received=1024, - ) - # complete -> sync_queued - await writer.append_state_transition(path, IngestState.SYNC_QUEUED, host=_HOST) - # sync_queued -> sync_verified - await writer.append_state_transition( - path, - IngestState.SYNC_VERIFIED, - host=_HOST, - nas_path="//nas/run", - checksum_file=".exlab-wizard/checksums.sha256", - ) - # sync_verified -> cleared - final = await writer.append_state_transition(path, IngestState.CLEARED, host=_HOST) - - assert final.current_state == IngestState.CLEARED.value - states_in_history = [h["state"] for h in final.history] - assert states_in_history == [ - IngestState.STAGING.value, # from initial payload - IngestState.COMPLETE.value, - IngestState.SYNC_QUEUED.value, - IngestState.SYNC_VERIFIED.value, - IngestState.CLEARED.value, - ] - # Inspect the on-disk file to make sure each entry survived the - # tmp+replace cycle and not just the in-memory return value. - on_disk = await writer.read_ingest(path) - assert [h["state"] for h in on_disk.history] == states_in_history - - -# --------------------------------------------------------------------------- -# Concurrent appends do not lose entries (FileLock contract) -# --------------------------------------------------------------------------- - - -async def test_concurrent_appends_serialize_via_filelock(tmp_path: Path) -> None: - """Five concurrent tasks attempt to transition the same file. - - Only one transition is legal from any single source state, so we expect - one task to succeed (staging -> complete) and the rest to raise - ``ValueError`` because the file is already past ``staging``. The point - of the test is that the lock prevents corruption: regardless of - interleaving, the file ends in a coherent state with exactly one - ``complete`` entry appended (no torn writes, no lost entries). - """ - writer = IngestWriter() - path = _ingest_path(tmp_path) - await writer.write_ingest(path, _make_payload()) - - async def attempt() -> str: - try: - await writer.append_state_transition( - path, - IngestState.COMPLETE, - host=_HOST, - files_received=1, - bytes_received=1, - ) - except ValueError: - return "rejected" - else: - return "ok" - - results = await asyncio.gather(*(attempt() for _ in range(5))) - assert results.count("ok") == 1 - assert results.count("rejected") == 4 - - final = await writer.read_ingest(path) - assert final.current_state == IngestState.COMPLETE.value - # Exactly one complete entry got appended; no duplicates and no torn - # writes left the file in an indecipherable state. - complete_entries = [h for h in final.history if h["state"] == IngestState.COMPLETE.value] - assert len(complete_entries) == 1 - - -async def test_concurrent_walk_through_state_machine_records_all_transitions( - tmp_path: Path, -) -> None: - """Five tasks each take one valid forward step, dispatched in parallel. - - Each task waits to find the file in *its* expected source state before - transitioning. The test asserts the final file has exactly five - additional history entries (one per task) in the correct order, proving - the FileLock ordered the read-mutate-write cycles correctly. - """ - writer = IngestWriter() - path = _ingest_path(tmp_path) - await writer.write_ingest(path, _make_payload()) - - async def wait_then_transition( - from_state: IngestState, - to_state: IngestState, - ) -> None: - # Spin until the file's current_state matches our source. Yields - # control so other tasks can advance the file. Bounded retry count - # so a stuck test fails fast rather than hanging. - for _ in range(200): - current = await writer.read_ingest(path) - if current.current_state == from_state.value: - break - await asyncio.sleep(0.01) - await writer.append_state_transition(path, to_state, host=_HOST) - - # The five forward transitions covering the full lifecycle. - transitions = [ - (IngestState.STAGING, IngestState.COMPLETE), - (IngestState.COMPLETE, IngestState.SYNC_QUEUED), - (IngestState.SYNC_QUEUED, IngestState.SYNC_VERIFIED), - (IngestState.SYNC_VERIFIED, IngestState.CLEARED), - ] - await asyncio.gather( - *(wait_then_transition(src, dst) for src, dst in transitions), - ) - - final = await writer.read_ingest(path) - assert final.current_state == IngestState.CLEARED.value - # Initial staging entry from _make_payload() + 4 transitions = 5 entries. - assert [h["state"] for h in final.history] == [ - IngestState.STAGING.value, - IngestState.COMPLETE.value, - IngestState.SYNC_QUEUED.value, - IngestState.SYNC_VERIFIED.value, - IngestState.CLEARED.value, - ] - - -# --------------------------------------------------------------------------- -# Schema-major mismatch on read (§11.9.2) -# --------------------------------------------------------------------------- - - -async def test_read_ingest_raises_on_schema_major_mismatch(tmp_path: Path) -> None: - from exlab_wizard.errors import SchemaMajorMismatchError - - writer = IngestWriter() - path = _ingest_path(tmp_path) - path.parent.mkdir(parents=True, exist_ok=True) - # Hand-write a v2.0 ingest.json -- the writer is at v1.x, so reading - # this file MUST raise SchemaMajorMismatchError per §11.9.2. - path.write_bytes( - msgspec.json.encode( - { - "schema_version": "2.0", - "project_name": "x", - "equipment_id": "X", - "run_kind": "experimental", - "run_path": "X/x/Run_x", - "transport": "smb_mount", - "current_state": "staging", - "history": [], - }, - ), - ) - - with pytest.raises(SchemaMajorMismatchError) as info: - await writer.read_ingest(path) - assert info.value.expected_major == 1 - assert info.value.found == "2.0" - - -async def test_append_state_transition_raises_on_schema_major_mismatch( - tmp_path: Path, -) -> None: - from exlab_wizard.errors import SchemaMajorMismatchError - - writer = IngestWriter() - path = _ingest_path(tmp_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes( - msgspec.json.encode( - { - "schema_version": "2.0", - "project_name": "x", - "equipment_id": "X", - "run_kind": "experimental", - "run_path": "X/x/Run_x", - "transport": "smb_mount", - "current_state": "staging", - "history": [], - }, - ), - ) - - with pytest.raises(SchemaMajorMismatchError): - await writer.append_state_transition(path, IngestState.COMPLETE, host=_HOST) - - -# --------------------------------------------------------------------------- -# default_host() -# --------------------------------------------------------------------------- - - -def test_default_host_returns_socket_gethostname() -> None: - """The convenience exposed for tests + orchestrator bootstrap matches - ``socket.gethostname()`` exactly.""" - import socket - - from exlab_wizard.cache.ingest_writer import default_host - - assert default_host() == socket.gethostname() diff --git a/tests/unit/cache/test_sync_state_writer.py b/tests/unit/cache/test_sync_state_writer.py new file mode 100644 index 0000000..897a93b --- /dev/null +++ b/tests/unit/cache/test_sync_state_writer.py @@ -0,0 +1,328 @@ +"""Unit tests for ``exlab_wizard.cache.sync_state_writer``. + +Covers the operator-free per-file NAS sync design (2026-05-21): +``sync_state.json`` read/write, the freely-mutable per-file ``upsert``, +the ``keep_local`` toggle, ``mark_cleared``, and the pure ``rollup_state`` +derivation. Each mutating method is exercised end-to-end against a real +file on ``tmp_path`` so the atomic-replace codepath runs. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import msgspec +import pytest + +from exlab_wizard.api.schemas import FileSyncRecord, SyncStateJson +from exlab_wizard.cache.sync_state_writer import SyncStateWriter +from exlab_wizard.constants import SYNC_STATE_FILENAME, SYNC_STATE_JSON_VERSION, RunSyncState +from exlab_wizard.errors import SchemaMajorMismatchError +from exlab_wizard.paths import cache_dir + + +def _state_path(tmp_path: Path) -> Path: + return cache_dir(tmp_path) / SYNC_STATE_FILENAME + + +# --------------------------------------------------------------------------- +# read -- absent file +# --------------------------------------------------------------------------- + + +async def test_read_when_absent_returns_empty_state(tmp_path: Path) -> None: + writer = SyncStateWriter() + + state = await writer.read(tmp_path) + + assert isinstance(state, SyncStateJson) + assert state.schema_version == SYNC_STATE_JSON_VERSION + assert state.cleared_at is None + assert state.files == {} + # Reading must not create the file. + assert not _state_path(tmp_path).exists() + + +# --------------------------------------------------------------------------- +# upsert_file -- create then update +# --------------------------------------------------------------------------- + + +async def test_upsert_file_creates_a_record(tmp_path: Path) -> None: + writer = SyncStateWriter() + + state = await writer.upsert_file( + tmp_path, + "data/scan.tif", + synced_signature=(1024, 17_000_000_000), + verified_at="2026-05-21T10:00:00Z", + ) + + assert _state_path(tmp_path).exists() + record = state.files["data/scan.tif"] + assert record.synced_signature == (1024, 17_000_000_000) + assert record.verified_at == "2026-05-21T10:00:00Z" + assert record.keep_local is False + # Persisted to disk, not just held in memory. + on_disk = await writer.read(tmp_path) + assert on_disk.files["data/scan.tif"].synced_signature == (1024, 17_000_000_000) + + +async def test_upsert_file_updates_existing_record(tmp_path: Path) -> None: + writer = SyncStateWriter() + await writer.upsert_file(tmp_path, "data/scan.tif", synced_signature=(10, 20)) + + updated = await writer.upsert_file( + tmp_path, + "data/scan.tif", + synced_signature=(99, 100), + verified_at="2026-05-21T12:00:00Z", + ) + + record = updated.files["data/scan.tif"] + assert record.synced_signature == (99, 100) + assert record.verified_at == "2026-05-21T12:00:00Z" + # Still exactly one record -- update, not append. + assert list(updated.files) == ["data/scan.tif"] + + +async def test_upsert_file_preserves_keep_local(tmp_path: Path) -> None: + writer = SyncStateWriter() + await writer.set_keep_local(tmp_path, "data/scan.tif", True) + + state = await writer.upsert_file( + tmp_path, + "data/scan.tif", + synced_signature=(5, 6), + verified_at="2026-05-21T13:00:00Z", + ) + + record = state.files["data/scan.tif"] + assert record.keep_local is True + assert record.synced_signature == (5, 6) + assert record.verified_at == "2026-05-21T13:00:00Z" + + +# --------------------------------------------------------------------------- +# set_keep_local -- toggle and create-if-absent +# --------------------------------------------------------------------------- + + +async def test_set_keep_local_creates_record_if_absent(tmp_path: Path) -> None: + writer = SyncStateWriter() + + state = await writer.set_keep_local(tmp_path, "data/scan.tif", True) + + record = state.files["data/scan.tif"] + assert record.keep_local is True + assert record.synced_signature is None + assert record.verified_at is None + + +async def test_set_keep_local_toggles_and_preserves_sync_fields(tmp_path: Path) -> None: + writer = SyncStateWriter() + await writer.upsert_file( + tmp_path, + "data/scan.tif", + synced_signature=(7, 8), + verified_at="2026-05-21T14:00:00Z", + ) + + enabled = await writer.set_keep_local(tmp_path, "data/scan.tif", True) + assert enabled.files["data/scan.tif"].keep_local is True + # Sync fields untouched by the toggle. + assert enabled.files["data/scan.tif"].synced_signature == (7, 8) + assert enabled.files["data/scan.tif"].verified_at == "2026-05-21T14:00:00Z" + + disabled = await writer.set_keep_local(tmp_path, "data/scan.tif", False) + assert disabled.files["data/scan.tif"].keep_local is False + assert disabled.files["data/scan.tif"].synced_signature == (7, 8) + + +# --------------------------------------------------------------------------- +# mark_cleared +# --------------------------------------------------------------------------- + + +async def test_mark_cleared_sets_cleared_at(tmp_path: Path) -> None: + writer = SyncStateWriter() + await writer.upsert_file(tmp_path, "data/scan.tif", synced_signature=(1, 2)) + + state = await writer.mark_cleared(tmp_path) + + assert state.cleared_at is not None + assert state.cleared_at.endswith("Z") + # Files survive cleanup (tombstone visibility). + assert "data/scan.tif" in state.files + # Persisted. + on_disk = await writer.read(tmp_path) + assert on_disk.cleared_at == state.cleared_at + + +async def test_mark_cleared_on_absent_file_creates_state(tmp_path: Path) -> None: + writer = SyncStateWriter() + + state = await writer.mark_cleared(tmp_path) + + assert state.cleared_at is not None + assert _state_path(tmp_path).exists() + + +async def test_mutators_create_missing_exlab_wizard_dir(tmp_path: Path) -> None: + """Each blocking mutator mkdir's the run's ``.exlab-wizard/`` cache dir. + + S2 robustness: ``atomic_write_bytes`` / ``FileLock`` do not create the + parent dir, so a mutator targeting a run with no ``.exlab-wizard/`` + would raise. ``_ensure_cache_dir`` makes every mutator self-healing. + """ + writer = SyncStateWriter() + # set_keep_local on a run dir whose cache dir does not exist. + upsert_run = tmp_path / "run_a" + upsert_run.mkdir() + keep_run = tmp_path / "run_b" + keep_run.mkdir() + clear_run = tmp_path / "run_c" + clear_run.mkdir() + + await writer.upsert_file(upsert_run, "data.bin", synced_signature=(1, 2)) + await writer.set_keep_local(keep_run, "data.bin", True) + await writer.mark_cleared(clear_run) + + assert _state_path(upsert_run).exists() + assert _state_path(keep_run).exists() + assert _state_path(clear_run).exists() + assert writer.read_sync(keep_run).files["data.bin"].keep_local is True + + +# --------------------------------------------------------------------------- +# rollup_state -- pure derivation +# --------------------------------------------------------------------------- + + +def test_rollup_state_syncing_when_empty() -> None: + state = SyncStateJson(schema_version=SYNC_STATE_JSON_VERSION) + assert SyncStateWriter.rollup_state(state) is RunSyncState.SYNCING + + +def test_rollup_state_syncing_when_some_unverified() -> None: + state = SyncStateJson( + schema_version=SYNC_STATE_JSON_VERSION, + files={ + "a": FileSyncRecord(verified_at="2026-05-21T10:00:00Z"), + "b": FileSyncRecord(verified_at=None), + }, + ) + assert SyncStateWriter.rollup_state(state) is RunSyncState.SYNCING + + +def test_rollup_state_synced_when_all_verified() -> None: + state = SyncStateJson( + schema_version=SYNC_STATE_JSON_VERSION, + files={ + "a": FileSyncRecord(verified_at="2026-05-21T10:00:00Z"), + "b": FileSyncRecord(verified_at="2026-05-21T11:00:00Z"), + }, + ) + assert SyncStateWriter.rollup_state(state) is RunSyncState.SYNCED + + +def test_rollup_state_cleared_takes_precedence_over_unverified() -> None: + state = SyncStateJson( + schema_version=SYNC_STATE_JSON_VERSION, + cleared_at="2026-05-21T15:00:00Z", + files={"a": FileSyncRecord(verified_at=None)}, + ) + assert SyncStateWriter.rollup_state(state) is RunSyncState.CLEARED + + +# --------------------------------------------------------------------------- +# Struct round-trip +# --------------------------------------------------------------------------- + + +def test_sync_state_json_round_trips_through_msgspec() -> None: + state = SyncStateJson( + schema_version=SYNC_STATE_JSON_VERSION, + cleared_at="2026-05-21T16:00:00Z", + files={ + "data/scan.tif": FileSyncRecord( + synced_signature=(2048, 99_000_000_000), + verified_at="2026-05-21T10:00:00Z", + keep_local=True, + ), + "data/raw.bin": FileSyncRecord(), + }, + ) + + blob = msgspec.json.encode(state) + decoded = msgspec.json.decode(blob, type=SyncStateJson) + + assert decoded == state + assert decoded.files["data/scan.tif"].synced_signature == (2048, 99_000_000_000) + assert decoded.files["data/scan.tif"].keep_local is True + assert decoded.files["data/raw.bin"].synced_signature is None + + +# --------------------------------------------------------------------------- +# Schema-major mismatch on read (§11.9.2) +# --------------------------------------------------------------------------- + + +async def test_read_raises_on_schema_major_mismatch(tmp_path: Path) -> None: + """A file at a different schema major than the writer is rejected. + + The writer is at v1.x; a hand-written v2.0 ``sync_state.json`` MUST + raise ``SchemaMajorMismatchError`` rather than silently partial-parse. + """ + path = _state_path(tmp_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes( + msgspec.json.encode( + { + "schema_version": "2.0", + "cleared_at": None, + "files": {}, + }, + ), + ) + + writer = SyncStateWriter() + with pytest.raises(SchemaMajorMismatchError) as info: + await writer.read(tmp_path) + assert info.value.expected_major == 1 + assert info.value.found == "2.0" + + +# --------------------------------------------------------------------------- +# Concurrent upserts do not lose records (FileLock contract) +# --------------------------------------------------------------------------- + + +async def test_concurrent_upserts_on_distinct_files_both_survive(tmp_path: Path) -> None: + """Two concurrent ``upsert_file`` calls on the same run, distinct paths. + + The ``FileLock`` serializes the read-mutate-write cycle so neither + upsert clobbers the other's record -- both must be present afterward. + """ + writer = SyncStateWriter() + + await asyncio.gather( + writer.upsert_file( + tmp_path, + "data/scan_a.tif", + synced_signature=(10, 20), + verified_at="2026-05-22T10:00:00Z", + ), + writer.upsert_file( + tmp_path, + "data/scan_b.tif", + synced_signature=(30, 40), + verified_at="2026-05-22T11:00:00Z", + ), + ) + + state = await writer.read(tmp_path) + assert set(state.files) == {"data/scan_a.tif", "data/scan_b.tif"} + assert state.files["data/scan_a.tif"].synced_signature == (10, 20) + assert state.files["data/scan_b.tif"].synced_signature == (30, 40) diff --git a/tests/unit/config/test_loader.py b/tests/unit/config/test_loader.py index 04eaa6b..1123085 100644 --- a/tests/unit/config/test_loader.py +++ b/tests/unit/config/test_loader.py @@ -60,15 +60,11 @@ def test_load_config_complete_yaml() -> None: assert len(cfg.equipment) == 2 confocal = cfg.equipment[0] assert confocal.id == "CONFOCAL_01" - assert confocal.completeness_signal == "sentinel_file" - assert confocal.sentinel_filename == "acquisition_complete.flag" assert confocal.transport.type == "rclone" assert confocal.transport.rclone_remote == "lab-nas" flow = cfg.equipment[1] assert flow.id == "FLOW_01" - assert flow.completeness_signal == "manifest" - assert flow.manifest_filename == "run_manifest.json" assert flow.transport.type == "rsync_ssh" assert flow.transport.ssh_target == "labuser@nas01.lab.example" @@ -118,8 +114,6 @@ def test_load_config_validation_error_raises_config_error(tmp_path: Path) -> Non " label: x\n" " local_root: /tmp\n" " nas_root: /mnt\n" - " completeness_signal: sentinel_file\n" - " sentinel_filename: done.flag\n" " transport:\n" " type: rclone\n" " rclone_remote: r\n" @@ -253,8 +247,6 @@ def test_dump_config_round_trip() -> None: "label": "Confocal", "local_root": "/l", "nas_root": "/n", - "completeness_signal": "sentinel_file", - "sentinel_filename": "done.flag", "transport": { "type": "rclone", "rclone_remote": "lab-nas", diff --git a/tests/unit/config/test_models.py b/tests/unit/config/test_models.py index e4696aa..7709215 100644 --- a/tests/unit/config/test_models.py +++ b/tests/unit/config/test_models.py @@ -36,6 +36,7 @@ RsyncSshTransport, SyncConfig, ValidatorConfig, + config_with_equipment_appended, ) from exlab_wizard.errors import ConfigError @@ -65,9 +66,6 @@ def _rsync_transport_dict() -> dict: def _equipment_dict( *, equipment_id: str = "CONFOCAL_01", - completeness_signal: str = "sentinel_file", - sentinel_filename: str | None = "acquisition_complete.flag", - manifest_filename: str | None = None, transport: dict | None = None, ) -> dict: """Build a valid EquipmentConfig dict with sensible defaults.""" @@ -76,9 +74,6 @@ def _equipment_dict( "label": "Confocal Microscope 1", "local_root": "/data/lab", "nas_root": "//nas01/lab", - "completeness_signal": completeness_signal, - "sentinel_filename": sentinel_filename, - "manifest_filename": manifest_filename, "transport": transport or _rclone_transport_dict(), } @@ -114,9 +109,6 @@ def _full_config_dict() -> dict: "label": "Confocal Microscope 1", "local_root": "/data/lab", "nas_root": "//nas01/lab", - "completeness_signal": "sentinel_file", - "sentinel_filename": "acquisition_complete.flag", - "manifest_filename": None, "sync_mode": "nas", "transport": { "type": "rclone", @@ -140,9 +132,6 @@ def _full_config_dict() -> dict: "label": "Flow Cytometer 1", "local_root": "/data/lab", "nas_root": "/mnt/nas/lab", - "completeness_signal": "manifest", - "sentinel_filename": None, - "manifest_filename": "run_manifest.json", "sync_mode": "nas", "transport": { "type": "rsync_ssh", @@ -189,7 +178,13 @@ def _full_config_dict() -> dict: ], }, "plugins": {"allow_network": False}, - "sync": {"enabled": True, "retry_attempts": 3}, + "sync": { + "enabled": True, + "retry_attempts": 3, + "quiescence_minutes": 10, + "ignore_globs": ["*.partial", "*.tmp"], + "poll_interval_seconds": 120, + }, "orchestrator": { "label": "Lab Acquisition Station 01", "staging_root": "/staging", @@ -458,9 +453,6 @@ def test_equipment_transport_discriminates_on_type_rsync() -> None: eq = EquipmentConfig.model_validate( _equipment_dict( equipment_id="FLOW_01", - completeness_signal="manifest", - sentinel_filename=None, - manifest_filename="run_manifest.json", transport=_rsync_transport_dict(), ) ) @@ -539,24 +531,15 @@ def test_equipment_id_accepts_max_length() -> None: assert eq.id == at_max -def test_completeness_signal_sentinel_requires_filename() -> None: - with pytest.raises(ValidationError) as info: - EquipmentConfig.model_validate( - _equipment_dict(completeness_signal="sentinel_file", sentinel_filename=None) - ) - assert "sentinel_filename" in str(info.value) - - -def test_completeness_signal_manifest_requires_filename() -> None: - with pytest.raises(ValidationError) as info: - EquipmentConfig.model_validate( - _equipment_dict( - completeness_signal="manifest", - sentinel_filename=None, - manifest_filename=None, - ) - ) - assert "manifest_filename" in str(info.value) +def test_equipment_config_rejects_removed_completeness_fields() -> None: + """The operator-free quiescence redesign drops the per-equipment + completeness-signal fields; ``extra='forbid'`` now rejects them.""" + for stale_key in ("completeness_signal", "sentinel_filename", "manifest_filename"): + bad = _equipment_dict() + bad[stale_key] = "x" + with pytest.raises(ValidationError) as info: + EquipmentConfig.model_validate(bad) + assert stale_key in str(info.value) def test_equipment_label_must_be_non_empty() -> None: @@ -818,6 +801,9 @@ def test_sync_config_defaults() -> None: cfg = SyncConfig() assert cfg.enabled is True assert cfg.retry_attempts == 3 + assert cfg.quiescence_minutes == 10 + assert cfg.ignore_globs == ["*.partial", "*.tmp"] + assert cfg.poll_interval_seconds == 120 def test_sync_config_retry_attempts_non_negative() -> None: @@ -825,6 +811,35 @@ def test_sync_config_retry_attempts_non_negative() -> None: SyncConfig(retry_attempts=-1) +def test_sync_config_quiescence_minutes_at_least_one() -> None: + with pytest.raises(ValidationError): + SyncConfig(quiescence_minutes=0) + + +def test_sync_config_poll_interval_seconds_at_least_one() -> None: + with pytest.raises(ValidationError): + SyncConfig(poll_interval_seconds=0) + + +def test_sync_config_accepts_custom_quiescence_settings() -> None: + cfg = SyncConfig( + quiescence_minutes=30, + ignore_globs=["*.lock"], + poll_interval_seconds=60, + ) + assert cfg.quiescence_minutes == 30 + assert cfg.ignore_globs == ["*.lock"] + assert cfg.poll_interval_seconds == 60 + + +def test_sync_config_ignore_globs_default_is_independent_per_instance() -> None: + # default_factory: mutating one instance's list must not bleed into another. + first = SyncConfig() + first.ignore_globs.append("*.bak") + second = SyncConfig() + assert second.ignore_globs == ["*.partial", "*.tmp"] + + # --------------------------------------------------------------------------- # OrchestratorStagingCleanup # --------------------------------------------------------------------------- @@ -895,9 +910,6 @@ def test_distinct_equipment_ids_accepted() -> None: _equipment_dict(equipment_id="CONFOCAL_01"), _equipment_dict( equipment_id="FLOW_01", - completeness_signal="manifest", - sentinel_filename=None, - manifest_filename="run_manifest.json", transport=_rsync_transport_dict(), ), ], @@ -989,3 +1001,46 @@ def test_round_trip_preserves_bandwidth_alias_for_from() -> None: assert schedule[0]["from"] == "08:00" assert schedule[0]["to"] == "18:00" assert "from_" not in schedule[0] + + +# --------------------------------------------------------------------------- +# config_with_equipment_appended (Redesign §6 -- shared append helper) +# --------------------------------------------------------------------------- + + +def test_config_with_equipment_appended_seeds_from_none() -> None: + """A ``None`` config (fresh install) yields a default Config + the device.""" + eq = EquipmentConfig.model_validate(_equipment_dict(equipment_id="CONFOCAL_01")) + result = config_with_equipment_appended(None, eq) + assert isinstance(result, Config) + assert [e.id for e in result.equipment] == ["CONFOCAL_01"] + + +def test_config_with_equipment_appended_preserves_existing_state() -> None: + """Appending keeps prior equipment and other config sections, unmutated.""" + first = EquipmentConfig.model_validate(_equipment_dict(equipment_id="CONFOCAL_01")) + base = Config(equipment=[first], logging=LoggingConfig(level="DEBUG")) + second = EquipmentConfig.model_validate( + _equipment_dict( + equipment_id="FLOW_02", + transport={ + "type": "rclone", + "rclone_remote": "lab-nas", + "rclone_remote_path": "lab/FLOW_02", + }, + ) + ) + result = config_with_equipment_appended(base, second) + assert [e.id for e in result.equipment] == ["CONFOCAL_01", "FLOW_02"] + assert result.logging.level == "DEBUG" + # The input config is copied, never mutated in place. + assert [e.id for e in base.equipment] == ["CONFOCAL_01"] + + +def test_config_with_equipment_appended_rejects_duplicate_id() -> None: + """A device whose id already exists raises ConfigError, not a silent merge.""" + existing = EquipmentConfig.model_validate(_equipment_dict(equipment_id="CONFOCAL_01")) + base = Config(equipment=[existing]) + dupe = EquipmentConfig.model_validate(_equipment_dict(equipment_id="CONFOCAL_01")) + with pytest.raises(ConfigError, match="CONFOCAL_01"): + config_with_equipment_appended(base, dupe) diff --git a/tests/unit/constants/test_enums.py b/tests/unit/constants/test_enums.py index 7a7f34d..cb08c9c 100644 --- a/tests/unit/constants/test_enums.py +++ b/tests/unit/constants/test_enums.py @@ -136,21 +136,16 @@ def test_lims_project_source_values() -> None: } -def test_ingest_state_values() -> None: - # Backend Spec §13.3. - assert issubclass(enums.IngestState, StrEnum) - assert enums.IngestState.STAGING.value == "staging" - assert enums.IngestState.COMPLETE.value == "complete" - assert enums.IngestState.SYNC_QUEUED.value == "sync_queued" - assert enums.IngestState.SYNC_VERIFIED.value == "sync_verified" - assert enums.IngestState.CLEARED.value == "cleared" - assert {m.value for m in enums.IngestState} == { - "staging", - "complete", - "sync_queued", - "sync_verified", - "cleared", - } +def test_ingest_state_enum_is_removed() -> None: + # The operator-free per-file NAS sync redesign (2026-05-21) removed the + # five-state ``IngestState`` machine along with ``ingest.json``. + assert not hasattr(enums, "IngestState") + + +def test_run_sync_state_values() -> None: + # Operator-free per-file NAS sync design (2026-05-21) -- derived rollup. + assert issubclass(enums.RunSyncState, StrEnum) + assert {m.value for m in enums.RunSyncState} == {"syncing", "synced", "cleared"} def test_setup_state_values() -> None: @@ -181,12 +176,9 @@ def test_transport_type_values() -> None: assert {m.value for m in enums.TransportType} == {"rclone", "rsync_ssh"} -def test_completeness_signal_values() -> None: - # Backend Spec §9, §13.5. - assert issubclass(enums.CompletenessSignal, StrEnum) - assert enums.CompletenessSignal.SENTINEL_FILE.value == "sentinel_file" - assert enums.CompletenessSignal.MANIFEST.value == "manifest" - assert {m.value for m in enums.CompletenessSignal} == {"sentinel_file", "manifest"} +def test_completeness_signal_enum_removed() -> None: + # The operator-free quiescence redesign removes CompletenessSignal. + assert not hasattr(enums, "CompletenessSignal") def test_staging_cleanup_mode_values() -> None: @@ -335,10 +327,9 @@ def test_enums_re_exported_from_package() -> None: assert constants.RunScope is enums.RunScope assert constants.LIMSProjectStatus is enums.LIMSProjectStatus assert constants.LIMSProjectSource is enums.LIMSProjectSource - assert constants.IngestState is enums.IngestState + assert constants.RunSyncState is enums.RunSyncState assert constants.SetupState is enums.SetupState assert constants.TransportType is enums.TransportType - assert constants.CompletenessSignal is enums.CompletenessSignal assert constants.StagingCleanupMode is enums.StagingCleanupMode assert constants.PluginStatus is enums.PluginStatus assert constants.CreationLevel is enums.CreationLevel diff --git a/tests/unit/constants/test_filenames.py b/tests/unit/constants/test_filenames.py index b8be3ff..f23d90a 100644 --- a/tests/unit/constants/test_filenames.py +++ b/tests/unit/constants/test_filenames.py @@ -29,9 +29,9 @@ def test_equipment_json_name() -> None: assert filenames.EQUIPMENT_JSON_NAME == "equipment.json" -def test_ingest_json_name() -> None: - # Backend Spec §13.4. - assert filenames.INGEST_JSON_NAME == "ingest.json" +def test_sync_state_filename() -> None: + # Operator-free per-file NAS sync design (2026-05-21). + assert filenames.SYNC_STATE_FILENAME == "sync_state.json" def test_test_runs_json_name() -> None: @@ -111,7 +111,7 @@ def test_filenames_re_exported_from_package() -> None: assert constants.CREATION_JSON_NAME == "creation.json" assert constants.README_FIELDS_JSON_NAME == "readme_fields.json" assert constants.EQUIPMENT_JSON_NAME == "equipment.json" - assert constants.INGEST_JSON_NAME == "ingest.json" + assert constants.SYNC_STATE_FILENAME == "sync_state.json" assert constants.TEST_RUNS_JSON_NAME == "test_runs.json" assert constants.ANSWERS_FILE_NAME == ".exlab-answers.yml" assert constants.LOG_FILE_TEMPLATE == "wizard.{hostname}.log" diff --git a/tests/unit/constants/test_schema_versions.py b/tests/unit/constants/test_schema_versions.py index 43d5021..4cf0066 100644 --- a/tests/unit/constants/test_schema_versions.py +++ b/tests/unit/constants/test_schema_versions.py @@ -20,9 +20,9 @@ def test_readme_fields_json_version_is_pinned() -> None: assert schema_versions.README_FIELDS_JSON_VERSION == "1.1" -def test_ingest_json_version_is_pinned() -> None: - # Backend Spec §13.4. - assert schema_versions.INGEST_JSON_VERSION == "1.1" +def test_sync_state_json_version_is_pinned() -> None: + # Operator-free per-file NAS sync design (2026-05-21). + assert schema_versions.SYNC_STATE_JSON_VERSION == "1.0" def test_equipment_json_version_is_pinned() -> None: @@ -51,7 +51,7 @@ def test_all_schema_versions_are_strings() -> None: for name in ( "CREATION_JSON_VERSION", "README_FIELDS_JSON_VERSION", - "INGEST_JSON_VERSION", + "SYNC_STATE_JSON_VERSION", "EQUIPMENT_JSON_VERSION", "TEST_RUNS_JSON_VERSION", "OFFLINE_CATALOGUE_VERSION", @@ -67,7 +67,7 @@ def test_schema_versions_re_exported_from_package() -> None: assert constants.CREATION_JSON_VERSION == "1.9" assert constants.README_FIELDS_JSON_VERSION == "1.1" - assert constants.INGEST_JSON_VERSION == "1.1" + assert constants.SYNC_STATE_JSON_VERSION == "1.0" assert constants.EQUIPMENT_JSON_VERSION == "1.0" assert constants.TEST_RUNS_JSON_VERSION == "1.0" assert constants.OFFLINE_CATALOGUE_VERSION == "1.0" diff --git a/tests/unit/orchestrator/test_cleanup.py b/tests/unit/orchestrator/test_cleanup.py deleted file mode 100644 index 48a1f3a..0000000 --- a/tests/unit/orchestrator/test_cleanup.py +++ /dev/null @@ -1,395 +0,0 @@ -"""Unit tests for ``exlab_wizard.orchestrator.cleanup``. - -Backend Spec §13.7. Covers the manual / scheduled policy decisions, -the on-disk delete, and the file-count + bytes-freed accounting. -""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta -from pathlib import Path - -import msgspec - -from exlab_wizard.api.schemas import IngestJson -from exlab_wizard.cache.ingest_writer import IngestWriter -from exlab_wizard.config.models import ( - Config, - OrchestratorConfig, - OrchestratorStagingCleanup, -) -from exlab_wizard.constants import ( - CACHE_DIR_NAME, - INGEST_JSON_NAME, - INGEST_JSON_VERSION, - IngestState, - StagingCleanupMode, -) -from exlab_wizard.orchestrator.cleanup import ( - cleanup_eligible, - clear_all_verified, - clear_run, - freed_bytes_and_count, -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_ingest(*, current_state: IngestState, history: list[dict]) -> IngestJson: - return msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "Test Project", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": "EQ1/PROJ-0001/Run_2026-04-17T14-32-00", - "transport": "smb_mount", - "current_state": current_state.value, - "history": history, - }, - type=IngestJson, - ) - - -def _orchestrator_config(*, mode: str, retain_hours: int = 24) -> Config: - return Config( - orchestrator=OrchestratorConfig( - label="ORCH-01", - staging_root="/staging", - staging_cleanup=OrchestratorStagingCleanup( - mode=mode, - retain_hours=retain_hours, - ), - ), - ) - - -# --------------------------------------------------------------------------- -# cleanup_eligible -- manual mode -# --------------------------------------------------------------------------- - - -def test_cleanup_eligible_manual_mode_always_returns_false() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.MANUAL.value) - ingest = _make_ingest( - current_state=IngestState.SYNC_VERIFIED, - history=[ - {"state": IngestState.SYNC_VERIFIED.value, "at": "2026-04-01T10:00:00Z"}, - ], - ) - # Even with a sync_verified entry from years ago, manual mode never auto-clears. - assert cleanup_eligible(ingest=ingest, config=config) is False - - -def test_cleanup_eligible_pre_sync_verified_states_return_false() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value) - for state in ( - IngestState.STAGING, - IngestState.COMPLETE, - IngestState.SYNC_QUEUED, - ): - ingest = _make_ingest( - current_state=state, history=[{"state": state.value, "at": "2026-04-01T00:00:00Z"}] - ) - assert cleanup_eligible(ingest=ingest, config=config) is False, state - - -def test_cleanup_eligible_cleared_state_returns_false() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value) - ingest = _make_ingest( - current_state=IngestState.CLEARED, - history=[{"state": IngestState.CLEARED.value, "at": "2026-04-01T00:00:00Z"}], - ) - assert cleanup_eligible(ingest=ingest, config=config) is False - - -# --------------------------------------------------------------------------- -# cleanup_eligible -- scheduled mode -# --------------------------------------------------------------------------- - - -def test_cleanup_eligible_scheduled_within_retain_window_returns_false() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value, retain_hours=24) - now = datetime(2026, 4, 17, 12, 0, 0, tzinfo=UTC) - # verified 1 hour ago -- still inside the retain window. - verified_at = (now - timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ") - ingest = _make_ingest( - current_state=IngestState.SYNC_VERIFIED, - history=[ - {"state": IngestState.STAGING.value, "at": "2026-04-01T00:00:00Z"}, - {"state": IngestState.COMPLETE.value, "at": "2026-04-01T01:00:00Z"}, - {"state": IngestState.SYNC_QUEUED.value, "at": "2026-04-01T02:00:00Z"}, - {"state": IngestState.SYNC_VERIFIED.value, "at": verified_at}, - ], - ) - assert cleanup_eligible(ingest=ingest, config=config, now_utc=now) is False - - -def test_cleanup_eligible_scheduled_after_retain_window_returns_true() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value, retain_hours=24) - now = datetime(2026, 4, 17, 12, 0, 0, tzinfo=UTC) - # verified 25 hours ago -- past the retain window. - verified_at = (now - timedelta(hours=25)).strftime("%Y-%m-%dT%H:%M:%SZ") - ingest = _make_ingest( - current_state=IngestState.SYNC_VERIFIED, - history=[ - {"state": IngestState.SYNC_VERIFIED.value, "at": verified_at}, - ], - ) - assert cleanup_eligible(ingest=ingest, config=config, now_utc=now) is True - - -def test_cleanup_eligible_scheduled_at_exact_retain_boundary_returns_true() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value, retain_hours=24) - now = datetime(2026, 4, 17, 12, 0, 0, tzinfo=UTC) - verified_at = (now - timedelta(hours=24)).strftime("%Y-%m-%dT%H:%M:%SZ") - ingest = _make_ingest( - current_state=IngestState.SYNC_VERIFIED, - history=[{"state": IngestState.SYNC_VERIFIED.value, "at": verified_at}], - ) - assert cleanup_eligible(ingest=ingest, config=config, now_utc=now) is True - - -def test_cleanup_eligible_returns_false_when_history_lacks_sync_verified_entry() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value) - # Defensive: somehow the file claims sync_verified state but no history entry. - ingest = _make_ingest( - current_state=IngestState.SYNC_VERIFIED, - history=[{"state": IngestState.STAGING.value, "at": "2026-04-01T00:00:00Z"}], - ) - assert cleanup_eligible(ingest=ingest, config=config) is False - - -def test_cleanup_eligible_handles_malformed_timestamp() -> None: - config = _orchestrator_config(mode=StagingCleanupMode.SCHEDULED.value) - ingest = _make_ingest( - current_state=IngestState.SYNC_VERIFIED, - history=[{"state": IngestState.SYNC_VERIFIED.value, "at": "not-an-iso-timestamp"}], - ) - assert cleanup_eligible(ingest=ingest, config=config) is False - - -# --------------------------------------------------------------------------- -# clear_run -# --------------------------------------------------------------------------- - - -async def _seed_staged_run(tmp_path: Path) -> tuple[Path, IngestWriter, Config]: - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"abcd" * 256) - (run_dir / "more.bin").write_bytes(b"x" * 1024) - cache_dir = run_dir / CACHE_DIR_NAME - cache_dir.mkdir() - writer = IngestWriter() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "Test Project", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": "EQ1/PROJ-0001/Run_2026-04-17T14-32-00", - "transport": "smb_mount", - "current_state": IngestState.SYNC_VERIFIED.value, - "history": [ - {"state": IngestState.STAGING.value, "at": "2026-04-17T14:00:00Z", "host": "h"}, - {"state": IngestState.COMPLETE.value, "at": "2026-04-17T14:30:00Z", "host": "h"}, - {"state": IngestState.SYNC_QUEUED.value, "at": "2026-04-17T14:31:00Z", "host": "h"}, - { - "state": IngestState.SYNC_VERIFIED.value, - "at": "2026-04-17T14:32:00Z", - "host": "h", - }, - ], - }, - type=IngestJson, - ) - await writer.write_ingest(cache_dir / INGEST_JSON_NAME, payload) - config = _orchestrator_config(mode=StagingCleanupMode.MANUAL.value) - return run_dir, writer, config - - -async def test_clear_run_deletes_staging_directory_and_returns_counts(tmp_path: Path) -> None: - run_dir, writer, config = await _seed_staged_run(tmp_path) - - files, bytes_freed = await clear_run(run_dir, config=config, ingest_writer=writer) - - assert not run_dir.exists() - # Two data files + the ingest.json -- the cleared entry was appended - # before the rmtree so the count picks up the on-disk file before - # it is removed. - assert files >= 2 - assert bytes_freed >= 1024 + 1024 # at least the two data files - - -async def test_clear_run_is_idempotent_when_directory_missing(tmp_path: Path) -> None: - config = _orchestrator_config(mode=StagingCleanupMode.MANUAL.value) - writer = IngestWriter() - nonexistent = tmp_path / "missing" / "Run_2026-04-17T14-32-00" - files, bytes_freed = await clear_run( - nonexistent, - config=config, - ingest_writer=writer, - ) - assert files == 0 - assert bytes_freed == 0 - - -async def test_clear_run_skips_ingest_write_when_no_ingest_file(tmp_path: Path) -> None: - """A defensive run without a staged ingest.json still gets deleted.""" - config = _orchestrator_config(mode=StagingCleanupMode.MANUAL.value) - writer = IngestWriter() - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_x" - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"hello") - files, bytes_freed = await clear_run( - run_dir, - config=config, - ingest_writer=writer, - ) - assert files == 1 - assert bytes_freed == len(b"hello") - assert not run_dir.exists() - - -def test_freed_bytes_and_count_returns_zero_for_missing_path(tmp_path: Path) -> None: - files, total = freed_bytes_and_count(tmp_path / "nope") - assert files == 0 - assert total == 0 - - -def test_freed_bytes_and_count_includes_all_nested_files(tmp_path: Path) -> None: - (tmp_path / "a.bin").write_bytes(b"x" * 10) - (tmp_path / "sub").mkdir() - (tmp_path / "sub" / "b.bin").write_bytes(b"y" * 25) - files, total = freed_bytes_and_count(tmp_path) - assert files == 2 - assert total == 35 - - -# --------------------------------------------------------------------------- -# clear_all_verified -- Redesign §4.6 bulk action -# --------------------------------------------------------------------------- - - -async def _seed_run_in_state( - staging_root: Path, - *, - equipment_id: str, - project_name: str, - run_dir_name: str, - state: IngestState, - writer: IngestWriter, -) -> Path: - """Helper: create a staged run directory with an ingest.json in ``state``.""" - run_dir = staging_root / equipment_id / project_name / run_dir_name - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"x" * 512) - cache_dir = run_dir / CACHE_DIR_NAME - cache_dir.mkdir() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": project_name, - "equipment_id": equipment_id, - "run_kind": "experimental", - "run_path": f"{equipment_id}/{project_name}/{run_dir_name}", - "transport": "smb_mount", - "current_state": state.value, - "history": [ - {"state": state.value, "at": "2026-04-17T14:32:00Z", "host": "h"}, - ], - }, - type=IngestJson, - ) - await writer.write_ingest(cache_dir / INGEST_JSON_NAME, payload) - return run_dir - - -async def test_clear_all_verified_clears_only_sync_verified_rows(tmp_path: Path) -> None: - writer = IngestWriter() - config = Config( - orchestrator=OrchestratorConfig( - label="ORCH-01", - staging_root=str(tmp_path), - staging_cleanup=OrchestratorStagingCleanup( - mode=StagingCleanupMode.MANUAL.value, - ), - ), - ) - - verified_a = await _seed_run_in_state( - tmp_path, - equipment_id="EQ1", - project_name="PROJ-A", - run_dir_name="Run_2026-05-01", - state=IngestState.SYNC_VERIFIED, - writer=writer, - ) - verified_b = await _seed_run_in_state( - tmp_path, - equipment_id="EQ1", - project_name="PROJ-A", - run_dir_name="Run_2026-05-02", - state=IngestState.SYNC_VERIFIED, - writer=writer, - ) - staging = await _seed_run_in_state( - tmp_path, - equipment_id="EQ2", - project_name="PROJ-B", - run_dir_name="Run_2026-05-03", - state=IngestState.STAGING, - writer=writer, - ) - - cleared = await clear_all_verified(config=config, ingest_writer=writer) - - # Only the two SYNC_VERIFIED rows are cleared; the STAGING row stays. - assert {Path(p) for p in cleared} == {verified_a, verified_b} - assert not verified_a.exists() - assert not verified_b.exists() - assert staging.exists() - - -async def test_clear_all_verified_returns_empty_list_when_no_verified_runs( - tmp_path: Path, -) -> None: - writer = IngestWriter() - config = Config( - orchestrator=OrchestratorConfig( - label="ORCH-01", - staging_root=str(tmp_path), - staging_cleanup=OrchestratorStagingCleanup( - mode=StagingCleanupMode.MANUAL.value, - ), - ), - ) - await _seed_run_in_state( - tmp_path, - equipment_id="EQ1", - project_name="PROJ-A", - run_dir_name="Run_X", - state=IngestState.STAGING, - writer=writer, - ) - cleared = await clear_all_verified(config=config, ingest_writer=writer) - assert cleared == [] - - -async def test_clear_all_verified_handles_empty_staging_root(tmp_path: Path) -> None: - writer = IngestWriter() - config = Config( - orchestrator=OrchestratorConfig( - label="ORCH-01", - staging_root=str(tmp_path / "empty"), - staging_cleanup=OrchestratorStagingCleanup( - mode=StagingCleanupMode.MANUAL.value, - ), - ), - ) - # staging_root doesn't even exist; list_staged_runs returns []. - cleared = await clear_all_verified(config=config, ingest_writer=writer) - assert cleared == [] diff --git a/tests/unit/orchestrator/test_quiescence_poller.py b/tests/unit/orchestrator/test_quiescence_poller.py new file mode 100644 index 0000000..989b652 --- /dev/null +++ b/tests/unit/orchestrator/test_quiescence_poller.py @@ -0,0 +1,369 @@ +"""Unit tests for ``exlab_wizard.orchestrator.quiescence_poller``. + +The :class:`QuiescenceSyncPoller` is driven synchronously through +:meth:`poll_once` with an injected monotonic clock so the per-file settle +window can be exercised deterministically without real time passing. + +Coverage: + +* a file becomes quiet only after the settle window has elapsed across + the poller's own sweeps; +* ``sync.ignore_globs`` files never make a run eligible; +* discovery spans both a staging-mode run and a ``nas``-mode run; +* per-file eligibility: a file matching its recorded ``synced_signature`` + is not re-enqueued, and a file modified after a recorded sync becomes + eligible again. +""" + +from __future__ import annotations + +from pathlib import Path + +from exlab_wizard.cache.sync_state_writer import SyncStateWriter +from exlab_wizard.config.models import ( + BandwidthConfig, + Config, + EquipmentConfig, + OrchestratorConfig, + OrchestratorStagingTransport, + PathsConfig, + RcloneTransport, + SyncConfig, +) +from exlab_wizard.constants import RUNS_DIR_NAME, SyncMode +from exlab_wizard.orchestrator.quiescence_poller import QuiescenceSyncPoller + +# --------------------------------------------------------------------------- +# Stubs +# --------------------------------------------------------------------------- + + +class _StubNasSync: + """In-memory NAS-sync stub recording per-file enqueue calls.""" + + def __init__(self) -> None: + self.enqueued: list[tuple[Path, list[str]]] = [] + + async def enqueue(self, run_path: Path, files: list[str] | None = None) -> None: + self.enqueued.append((run_path, list(files or []))) + + @property + def enqueued_paths(self) -> list[Path]: + return [run_path for run_path, _ in self.enqueued] + + +# --------------------------------------------------------------------------- +# Config / fixture helpers +# --------------------------------------------------------------------------- + + +def _transport() -> RcloneTransport: + return RcloneTransport( + type="rclone", + rclone_remote="lab-nas", + rclone_remote_path="/srv/nas", + bandwidth=BandwidthConfig(), + ) + + +def _make_config( + *, + staging_root: Path | None = None, + nas_equipment_root: Path | None = None, + quiescence_minutes: int = 1, +) -> Config: + """Build a Config with an optional staging root and a nas-mode equipment.""" + equipment: list[EquipmentConfig] = [] + if nas_equipment_root is not None: + equipment.append( + EquipmentConfig( + id="EQNAS", + label="Nas Equipment", + local_root=str(nas_equipment_root), + nas_root="/nas", + sync_mode=SyncMode.NAS, + transport=_transport(), + ), + ) + local_root = staging_root or nas_equipment_root or Path("/tmp") + return Config( + paths=PathsConfig(local_root=str(local_root)), + equipment=equipment, + orchestrator=OrchestratorConfig( + label="ORCH", + staging_root=str(staging_root) if staging_root is not None else "", + ), + sync=SyncConfig(quiescence_minutes=quiescence_minutes), + ) + + +def _make_run(root: Path, equipment_id: str, *, with_file: bool = True) -> Path: + """Create a run-leaf directory mirroring the §13.2 staging layout.""" + run_dir = root / equipment_id / "PROJ-0001" / RUNS_DIR_NAME / "Run_2026-05-21T10-00-00" + run_dir.mkdir(parents=True) + if with_file: + (run_dir / "data.bin").write_bytes(b"payload" * 100) + return run_dir + + +def _poller(config: Config, nas_sync: _StubNasSync) -> QuiescenceSyncPoller: + return QuiescenceSyncPoller( + config=config, + nas_sync=nas_sync, + sync_state_writer=SyncStateWriter(), + ) + + +def _signature(path: Path) -> tuple[int, int]: + st = path.stat() + return (st.st_size, st.st_mtime_ns) + + +# --------------------------------------------------------------------------- +# Settle window +# --------------------------------------------------------------------------- + + +async def test_file_becomes_quiet_only_after_settle_window(tmp_path: Path) -> None: + """A file is enqueued only after its signature settles for the window.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) # 60s window + nas_sync = _StubNasSync() + poller = _poller(config, nas_sync) + run_dir = _make_run(tmp_path, "EQ1") + + # First sweep: file just observed -- not yet quiet. + enqueued = await poller.poll_once(now_monotonic=0.0) + assert enqueued == [] + assert nas_sync.enqueued == [] + + # 30s later: still inside the window. + enqueued = await poller.poll_once(now_monotonic=30.0) + assert enqueued == [] + + # 60s after first observation: window elapsed -> enqueued. + enqueued = await poller.poll_once(now_monotonic=60.0) + assert enqueued == [run_dir] + assert nas_sync.enqueued == [(run_dir, ["data.bin"])] + + +async def test_modified_file_resets_the_settle_window(tmp_path: Path) -> None: + """A signature change resets ``first_seen``; the window restarts.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + nas_sync = _StubNasSync() + poller = _poller(config, nas_sync) + run_dir = _make_run(tmp_path, "EQ1") + target = run_dir / "data.bin" + + await poller.poll_once(now_monotonic=0.0) + # Modify the file just before the window would elapse. + target.write_bytes(b"changed-payload" * 50) + await poller.poll_once(now_monotonic=50.0) + # 60s after the *first* observation -- but the file changed at 50s, so + # it is not yet quiet. + enqueued = await poller.poll_once(now_monotonic=60.0) + assert enqueued == [] + # 60s after the modification -> finally quiet. + enqueued = await poller.poll_once(now_monotonic=110.0) + assert enqueued == [run_dir] + + +# --------------------------------------------------------------------------- +# Ignore globs +# --------------------------------------------------------------------------- + + +async def test_ignore_glob_files_do_not_make_a_run_eligible(tmp_path: Path) -> None: + """A run holding only ignore-glob files never enqueues.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + nas_sync = _StubNasSync() + poller = _poller(config, nas_sync) + run_dir = _make_run(tmp_path, "EQ1", with_file=False) + # Default ignore_globs == ["*.partial", "*.tmp"]. + (run_dir / "upload.partial").write_bytes(b"in-progress") + (run_dir / "scratch.tmp").write_bytes(b"temp") + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + assert enqueued == [] + assert nas_sync.enqueued == [] + + +async def test_quiet_real_file_enqueues_despite_ignored_sibling(tmp_path: Path) -> None: + """A quiet non-ignored file enqueues even when ignored files coexist.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + nas_sync = _StubNasSync() + poller = _poller(config, nas_sync) + run_dir = _make_run(tmp_path, "EQ1") + (run_dir / "upload.partial").write_bytes(b"in-progress") + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + assert enqueued == [run_dir] + # Only the real file rides the enqueue; the ignored sibling never does. + assert nas_sync.enqueued == [(run_dir, ["data.bin"])] + + +# --------------------------------------------------------------------------- +# Discovery: staging-mode and nas-mode +# --------------------------------------------------------------------------- + + +async def test_discovers_both_staging_and_nas_mode_runs(tmp_path: Path) -> None: + """A sweep finds runs under staging_root AND under nas-mode local_root.""" + staging_root = tmp_path / "staging" + nas_root = tmp_path / "nas-local" + staging_root.mkdir() + nas_root.mkdir() + config = _make_config( + staging_root=staging_root, + nas_equipment_root=nas_root, + quiescence_minutes=1, + ) + nas_sync = _StubNasSync() + poller = _poller(config, nas_sync) + staging_run = _make_run(staging_root, "EQ1") + nas_run = _make_run(nas_root, "EQNAS") + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + assert set(enqueued) == {staging_run, nas_run} + assert set(nas_sync.enqueued_paths) == {staging_run, nas_run} + + +async def test_co_rooted_stage_mode_equipment_run_is_not_enqueued(tmp_path: Path) -> None: + """A ``stage``-mode equipment sharing one ``local_root`` with a + ``nas``-mode equipment must NOT have its runs swept into NAS sync -- + only the ``nas``-mode equipment's own subtree is walked.""" + shared_root = tmp_path / "lab-data" + shared_root.mkdir() + config = Config( + paths=PathsConfig(local_root=str(shared_root)), + equipment=[ + EquipmentConfig( + id="EQNAS", + label="Nas Equipment", + local_root=str(shared_root), + nas_root="/nas", + sync_mode=SyncMode.NAS, + transport=_transport(), + ), + EquipmentConfig( + id="EQSTAGE", + label="Stage Equipment", + local_root=str(shared_root), + nas_root="/nas", + sync_mode=SyncMode.STAGE, + orchestrator_staging_transport=OrchestratorStagingTransport( + type="smb_mount", + mount_point="/mnt/orch", + staging_subpath="staging", + ), + ), + ], + orchestrator=OrchestratorConfig(label="ORCH", staging_root=""), + sync=SyncConfig(quiescence_minutes=1), + ) + nas_sync = _StubNasSync() + poller = _poller(config, nas_sync) + nas_run = _make_run(shared_root, "EQNAS") + stage_run = _make_run(shared_root, "EQSTAGE") + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + # Only the nas-mode equipment's run is enqueued; the co-rooted + # stage-mode run reaches the NAS via the orchestrator staging area. + assert enqueued == [nas_run] + assert stage_run not in nas_sync.enqueued_paths + + +# --------------------------------------------------------------------------- +# Per-file eligibility against sync_state.json +# --------------------------------------------------------------------------- + + +async def test_file_matching_synced_signature_is_not_re_enqueued(tmp_path: Path) -> None: + """A quiet file already synced at its current signature is skipped.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + nas_sync = _StubNasSync() + writer = SyncStateWriter() + poller = QuiescenceSyncPoller(config=config, nas_sync=nas_sync, sync_state_writer=writer) + run_dir = _make_run(tmp_path, "EQ1") + target = run_dir / "data.bin" + + # Record the file as already synced at its current (size, mtime). + await writer.upsert_file( + run_dir, + "data.bin", + synced_signature=_signature(target), + verified_at="2026-05-21T10:00:00Z", + ) + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + # The only file matches its recorded signature -> nothing eligible. + assert enqueued == [] + assert nas_sync.enqueued == [] + + +async def test_file_modified_after_recorded_sync_becomes_eligible(tmp_path: Path) -> None: + """A file modified after its recorded sync re-enters the eligible set.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + nas_sync = _StubNasSync() + writer = SyncStateWriter() + poller = QuiescenceSyncPoller(config=config, nas_sync=nas_sync, sync_state_writer=writer) + run_dir = _make_run(tmp_path, "EQ1") + + # Record a stale signature (a sync that predates the current content). + await writer.upsert_file( + run_dir, + "data.bin", + synced_signature=(1, 1), + verified_at="2026-05-21T10:00:00Z", + ) + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + # Current signature differs from the recorded one -> eligible again. + assert enqueued == [run_dir] + assert nas_sync.enqueued == [(run_dir, ["data.bin"])] + + +async def test_only_changed_file_in_a_run_is_enqueued(tmp_path: Path) -> None: + """A run with a mix of synced + modified files enqueues only the dirty one.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + nas_sync = _StubNasSync() + writer = SyncStateWriter() + poller = QuiescenceSyncPoller(config=config, nas_sync=nas_sync, sync_state_writer=writer) + run_dir = _make_run(tmp_path, "EQ1", with_file=False) + clean = run_dir / "clean.bin" + dirty = run_dir / "dirty.bin" + clean.write_bytes(b"clean-data") + dirty.write_bytes(b"dirty-data") + + # clean.bin is recorded at its current signature; dirty.bin at a stale one. + await writer.upsert_file( + run_dir, "clean.bin", synced_signature=_signature(clean), verified_at="2026-05-21T10:00:00Z" + ) + await writer.upsert_file( + run_dir, "dirty.bin", synced_signature=(9, 9), verified_at="2026-05-21T10:00:00Z" + ) + + await poller.poll_once(now_monotonic=0.0) + enqueued = await poller.poll_once(now_monotonic=120.0) + assert enqueued == [run_dir] + assert nas_sync.enqueued == [(run_dir, ["dirty.bin"])] + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +async def test_start_then_stop_is_idempotent(tmp_path: Path) -> None: + """start()/stop() can be called repeatedly without error.""" + config = _make_config(staging_root=tmp_path, quiescence_minutes=1) + poller = _poller(config, _StubNasSync()) + await poller.start() + await poller.start() # idempotent + await poller.stop() + await poller.stop() # idempotent diff --git a/tests/unit/orchestrator/test_scan.py b/tests/unit/orchestrator/test_scan.py index 5fcddd7..ade6515 100644 --- a/tests/unit/orchestrator/test_scan.py +++ b/tests/unit/orchestrator/test_scan.py @@ -1,7 +1,7 @@ """Unit tests for ``exlab_wizard.orchestrator._scan``. The shared filesystem helpers used by both ``staging_query`` and -``staging_watcher``. Covers the §13.2 walk pattern and the +``quiescence_poller``. Covers the §13.2 walk pattern and the file-count / byte-total accounting (with and without the cache dir). """ diff --git a/tests/unit/orchestrator/test_staging_query.py b/tests/unit/orchestrator/test_staging_query.py index 323fa48..f8eae9a 100644 --- a/tests/unit/orchestrator/test_staging_query.py +++ b/tests/unit/orchestrator/test_staging_query.py @@ -1,44 +1,32 @@ """Unit tests for ``exlab_wizard.orchestrator.staging_query``. -Backend Spec §13.8. Verify the walker discovers run leaves, decodes -``ingest.json``, and returns rows sorted by last activity (most recent -first). +Backend Spec §13.8. The operator-free per-file NAS sync redesign +(2026-05-21) removed ``ingest.json``; the query discovers run leaves, +derives identity from the run path, and reports ``current_state`` as the +derived ``sync_state.json`` rollup (``syncing`` / ``synced`` / ``cleared``). +Rows are sorted by directory mtime, most recent first. """ from __future__ import annotations -from datetime import UTC, datetime, timedelta +import asyncio from pathlib import Path -import msgspec - -from exlab_wizard.api.schemas import IngestJson -from exlab_wizard.cache.ingest_writer import IngestWriter +from exlab_wizard.cache.sync_state_writer import SyncStateWriter from exlab_wizard.config.models import ( Config, OrchestratorConfig, OrchestratorStagingCleanup, ) -from exlab_wizard.constants import ( - CACHE_DIR_NAME, - INGEST_JSON_NAME, - INGEST_JSON_VERSION, - IngestState, -) -from exlab_wizard.orchestrator.staging_query import ( - StagedRunSummary, - list_staged_runs, -) +from exlab_wizard.constants import RUNS_DIR_NAME, TEST_RUNS_DIR_NAME +from exlab_wizard.orchestrator.staging_query import StagedRunSummary, list_staged_runs # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _config(staging_root: Path, *, enabled: bool = True) -> Config: - # ``enabled`` is accepted for backward compat with older test call - # sites; Redesign §3.1 makes the orchestrator pipeline always active. - _ = enabled +def _config(staging_root: Path) -> Config: return Config( orchestrator=OrchestratorConfig( label="ORCH", @@ -48,53 +36,30 @@ def _config(staging_root: Path, *, enabled: bool = True) -> Config: ) -async def _seed_run( +def _seed_run( staging_root: Path, *, equipment: str = "EQ1", project: str = "PROJ-0001", run_name: str = "Run_2026-04-17T14-32-00", test_run: bool = False, - state: IngestState = IngestState.STAGING, - last_at: str = "2026-04-17T14:30:00Z", extra_files: tuple[tuple[str, bytes], ...] = (("data.bin", b"abcd" * 256),), ) -> Path: - if test_run: - run_dir = staging_root / equipment / project / "TestRuns" / run_name - else: - run_dir = staging_root / equipment / project / run_name + marker = TEST_RUNS_DIR_NAME if test_run else RUNS_DIR_NAME + run_dir = staging_root / equipment / project / marker / run_name run_dir.mkdir(parents=True) for fname, fdata in extra_files: (run_dir / fname).write_bytes(fdata) - cache_dir = run_dir / CACHE_DIR_NAME - cache_dir.mkdir() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": project, - "equipment_id": equipment, - "run_kind": "test" if test_run else "experimental", - "run_path": str(run_dir), - "transport": "smb_mount", - "current_state": state.value, - "history": [ - {"state": state.value, "at": last_at, "host": "host"}, - ], - }, - type=IngestJson, - ) - writer = IngestWriter() - await writer.write_ingest(cache_dir / INGEST_JSON_NAME, payload) return run_dir # --------------------------------------------------------------------------- -# Disabled / missing-directory cases +# Missing-directory cases # --------------------------------------------------------------------------- -def test_list_staged_runs_returns_empty_when_orchestrator_disabled(tmp_path: Path) -> None: - config = _config(tmp_path, enabled=False) +def test_list_staged_runs_returns_empty_when_staging_root_unset() -> None: + config = Config(orchestrator=OrchestratorConfig(label="ORCH", staging_root="")) assert list_staged_runs(config=config) == [] @@ -103,30 +68,20 @@ def test_list_staged_runs_returns_empty_when_staging_root_missing(tmp_path: Path assert list_staged_runs(config=config) == [] -def test_list_staged_runs_skips_runs_without_ingest_json(tmp_path: Path) -> None: - # Stage a run without writing ingest.json (mid-push). - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"x") - config = _config(tmp_path) - assert list_staged_runs(config=config) == [] - - # --------------------------------------------------------------------------- # Happy path # --------------------------------------------------------------------------- -async def test_list_staged_runs_returns_summary_rows(tmp_path: Path) -> None: - await _seed_run(tmp_path) - config = _config(tmp_path) - - rows = list_staged_runs(config=config) +def test_list_staged_runs_returns_summary_rows(tmp_path: Path) -> None: + _seed_run(tmp_path) + rows = list_staged_runs(config=_config(tmp_path)) assert len(rows) == 1 row = rows[0] assert isinstance(row, StagedRunSummary) - assert row.current_state == IngestState.STAGING.value + # No sync_state.json yet -> rolls up to "syncing". + assert row.current_state == "syncing" assert row.equipment_id == "EQ1" assert row.project_name == "PROJ-0001" assert row.run_kind == "experimental" @@ -134,138 +89,114 @@ async def test_list_staged_runs_returns_summary_rows(tmp_path: Path) -> None: assert row.byte_total == 4 * 256 -async def test_list_staged_runs_includes_test_runs_under_TestRuns(tmp_path: Path) -> None: - await _seed_run( - tmp_path, - run_name="TestRun_2026-04-17T09-12-00", - test_run=True, - ) - config = _config(tmp_path) - - rows = list_staged_runs(config=config) +def test_list_staged_runs_includes_test_runs_under_TestRuns(tmp_path: Path) -> None: + _seed_run(tmp_path, run_name="TestRun_2026-04-17T09-12-00", test_run=True) + rows = list_staged_runs(config=_config(tmp_path)) assert len(rows) == 1 assert rows[0].run_kind == "test" -async def test_list_staged_runs_sorts_most_recent_first(tmp_path: Path) -> None: - await _seed_run( - tmp_path, - run_name="Run_2026-04-15T00-00-00", - last_at="2026-04-15T00:00:00Z", +def test_list_staged_runs_reports_syncing_when_a_file_is_unverified(tmp_path: Path) -> None: + """A run with an unverified tracked file rolls up to ``syncing``.""" + run_dir = _seed_run(tmp_path) + writer = SyncStateWriter() + asyncio.run(writer.upsert_file(run_dir, "data.bin", synced_signature=None, verified_at=None)) + rows = list_staged_runs(config=_config(tmp_path)) + assert rows[0].current_state == "syncing" + + +def test_list_staged_runs_reports_synced_when_all_files_verified(tmp_path: Path) -> None: + """Every tracked file verified -> ``current_state == 'synced'``.""" + run_dir = _seed_run(tmp_path) + writer = SyncStateWriter() + asyncio.run( + writer.upsert_file( + run_dir, "data.bin", synced_signature=(1024, 111), verified_at="2026-05-21T00:00:00Z" + ) ) - await _seed_run( - tmp_path, - project="PROJ-0002", - run_name="Run_2026-04-17T00-00-00", - last_at="2026-04-17T00:00:00Z", + rows = list_staged_runs(config=_config(tmp_path)) + assert rows[0].current_state == "synced" + + +def test_list_staged_runs_reports_cleared_after_mark_cleared(tmp_path: Path) -> None: + """A run whose ``sync_state.json`` carries ``cleared_at`` rolls up to ``cleared``.""" + run_dir = _seed_run(tmp_path) + writer = SyncStateWriter() + asyncio.run( + writer.upsert_file( + run_dir, "data.bin", synced_signature=(1024, 111), verified_at="2026-05-21T00:00:00Z" + ) ) - await _seed_run( - tmp_path, - project="PROJ-0003", - run_name="Run_2026-04-16T00-00-00", - last_at="2026-04-16T00:00:00Z", + asyncio.run(writer.mark_cleared(run_dir)) + rows = list_staged_runs(config=_config(tmp_path)) + assert rows[0].current_state == "cleared" + + +def test_list_staged_runs_remodified_file_flips_rollup_back_to_syncing(tmp_path: Path) -> None: + """A re-modified file (signature cleared -> unverified) flips synced back to syncing.""" + run_dir = _seed_run(tmp_path) + writer = SyncStateWriter() + # First fully verified. + asyncio.run( + writer.upsert_file( + run_dir, "data.bin", synced_signature=(1024, 111), verified_at="2026-05-21T00:00:00Z" + ) ) + assert list_staged_runs(config=_config(tmp_path))[0].current_state == "synced" + # The file is re-modified: the poller clears the verify mark for the + # re-eligible file, dropping the run rollup back to ``syncing``. + asyncio.run(writer.upsert_file(run_dir, "data.bin", synced_signature=None, verified_at=None)) + assert list_staged_runs(config=_config(tmp_path))[0].current_state == "syncing" + +def test_list_staged_runs_includes_runs_without_creation_metadata(tmp_path: Path) -> None: + """A run leaf with no cache metadata is still listed (ingest.json gone).""" + run_dir = tmp_path / "EQ1" / "PROJ-0001" / RUNS_DIR_NAME / "Run_2026-04-17T14-32-00" + run_dir.mkdir(parents=True) + (run_dir / "data.bin").write_bytes(b"x") rows = list_staged_runs(config=_config(tmp_path)) + assert len(rows) == 1 + assert rows[0].equipment_id == "EQ1" - assert [r.last_activity_at for r in rows] == [ - "2026-04-17T00:00:00Z", - "2026-04-16T00:00:00Z", - "2026-04-15T00:00:00Z", - ] +def test_list_staged_runs_sorts_most_recent_first(tmp_path: Path) -> None: + import os + import time -async def test_list_staged_runs_computes_elapsed_seconds(tmp_path: Path) -> None: - last_at = "2026-04-17T12:00:00Z" - await _seed_run(tmp_path, last_at=last_at) - config = _config(tmp_path) - now = datetime(2026, 4, 17, 12, 30, 0, tzinfo=UTC) + older = _seed_run(tmp_path, run_name="Run_2026-04-15T00-00-00") + newer = _seed_run(tmp_path, project="PROJ-0002", run_name="Run_2026-04-17T00-00-00") + # Force a deterministic mtime ordering. + base = time.time() + os.utime(older, (base - 200, base - 200)) + os.utime(newer, (base, base)) - rows = list_staged_runs(config=config, now_utc=now) + rows = list_staged_runs(config=_config(tmp_path)) + assert [r.path for r in rows] == [str(newer), str(older)] - assert rows[0].elapsed_seconds_since_last_activity == int(timedelta(minutes=30).total_seconds()) +def test_list_staged_runs_excludes_cache_dir_from_byte_total(tmp_path: Path) -> None: + """The .exlab-wizard subtree is metadata, not staged data.""" + from exlab_wizard.constants import CACHE_DIR_NAME -async def test_list_staged_runs_skips_unparsable_ingest_json(tmp_path: Path) -> None: - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) + run_dir = _seed_run(tmp_path) cache = run_dir / CACHE_DIR_NAME cache.mkdir() - (cache / INGEST_JSON_NAME).write_bytes(b"not json {{{") - config = _config(tmp_path) - rows = list_staged_runs(config=config) - assert rows == [] - - -async def test_list_staged_runs_excludes_cache_dir_from_byte_total(tmp_path: Path) -> None: - """The .exlab-wizard subtree is metadata, not staged data.""" - run_dir = await _seed_run(tmp_path) - # Add a junk file inside .exlab-wizard to ensure it's NOT counted. - (run_dir / CACHE_DIR_NAME / "junk.bin").write_bytes(b"a" * 1000) + (cache / "junk.bin").write_bytes(b"a" * 1000) rows = list_staged_runs(config=_config(tmp_path)) assert rows[0].byte_total == 4 * 256 # only the data.bin -async def test_list_staged_runs_uses_explicit_staging_root_param(tmp_path: Path) -> None: +def test_list_staged_runs_uses_explicit_staging_root_param(tmp_path: Path) -> None: other_root = tmp_path / "other" - await _seed_run(other_root) - # Config points at a different directory; explicit staging_root overrides. + _seed_run(other_root) config = _config(tmp_path / "ignored") rows = list_staged_runs(config=config, staging_root=other_root) assert len(rows) == 1 -async def test_list_staged_runs_falls_back_to_mtime_when_history_empty( - tmp_path: Path, -) -> None: - """An ingest.json with empty history uses the directory mtime as fallback.""" - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"x") - cache = run_dir / CACHE_DIR_NAME - cache.mkdir() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "PROJ-0001", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": str(run_dir), - "transport": "smb_mount", - "current_state": IngestState.STAGING.value, - "history": [], # explicit empty history - }, - type=IngestJson, - ) - await IngestWriter().write_ingest(cache / INGEST_JSON_NAME, payload) +def test_list_staged_runs_sets_last_activity_from_directory_mtime(tmp_path: Path) -> None: + _seed_run(tmp_path) rows = list_staged_runs(config=_config(tmp_path)) - assert len(rows) == 1 # The mtime-derived ISO string is non-empty. assert rows[0].last_activity_at - - -async def test_list_staged_runs_handles_history_entry_without_at_field( - tmp_path: Path, -) -> None: - """Defensive: a malformed history entry with no ``at`` falls back to mtime.""" - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"x") - cache = run_dir / CACHE_DIR_NAME - cache.mkdir() - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "PROJ-0001", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": str(run_dir), - "transport": "smb_mount", - "current_state": IngestState.STAGING.value, - "history": [{"state": IngestState.STAGING.value, "host": "host"}], # no ``at`` - }, - type=IngestJson, - ) - await IngestWriter().write_ingest(cache / INGEST_JSON_NAME, payload) - rows = list_staged_runs(config=_config(tmp_path)) - assert rows[0].last_activity_at # something fell-through + assert rows[0].elapsed_seconds_since_last_activity >= 0 diff --git a/tests/unit/orchestrator/test_staging_watcher.py b/tests/unit/orchestrator/test_staging_watcher.py deleted file mode 100644 index 6f946aa..0000000 --- a/tests/unit/orchestrator/test_staging_watcher.py +++ /dev/null @@ -1,773 +0,0 @@ -"""Unit tests for ``exlab_wizard.orchestrator.staging_watcher``. - -Backend Spec §13.3, §13.5, §13.7. Each five-state transition is driven -synchronously through :meth:`StagingWatcher.evaluate_run` so we can -assert the on-disk effect without spinning up the polling task. -""" - -from __future__ import annotations - -import asyncio -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import msgspec - -from exlab_wizard.api.schemas import ( - CreationJson, - IngestJson, -) -from exlab_wizard.cache.ingest_writer import IngestWriter -from exlab_wizard.config.models import ( - BandwidthConfig, - Config, - EquipmentConfig, - OrchestratorConfig, - OrchestratorStagingCleanup, - PathsConfig, - RcloneTransport, -) -from exlab_wizard.constants import ( - CACHE_DIR_NAME, - CREATION_JSON_NAME, - CREATION_JSON_VERSION, - INGEST_JSON_NAME, - INGEST_JSON_VERSION, - IngestState, - StagingCleanupMode, -) -from exlab_wizard.orchestrator.staging_watcher import StagingWatcher - -# --------------------------------------------------------------------------- -# Stubs -# --------------------------------------------------------------------------- - - -@dataclass -class StubHandle: - job_id: str = "job-1" - state: str = "queued" - run_path: str = "" - blocking_findings: tuple = () - - -class StubNasSync: - """Simple in-memory NAS sync client matching the protocol.""" - - def __init__(self) -> None: - self.enqueue_calls: list[Path] = [] - self.status_responses: dict[str, str] = {} - - async def enqueue(self, run_path: Path) -> StubHandle: - self.enqueue_calls.append(run_path) - return StubHandle(run_path=str(run_path)) - - async def status(self, run_path: Path) -> str: - return self.status_responses.get(str(run_path), "queued") - - -class StubCreationCache: - """Returns a hand-built CreationJson when read.""" - - def __init__(self, *, payload: CreationJson | None = None) -> None: - self._payload = payload - - async def read_creation_snapshot(self, path: Path) -> CreationJson: - if self._payload is None: - raise FileNotFoundError(path) - return self._payload - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -def _make_config( - staging_root: Path, - *, - completeness_signal: str = "sentinel_file", - sentinel_filename: str = "run_complete", - manifest_filename: str | None = None, - cleanup_mode: str = StagingCleanupMode.MANUAL.value, - retain_hours: int = 24, - enabled: bool = True, # accepted for backward compat; orchestrator pipeline is always on -) -> Config: - return Config( - paths=PathsConfig(local_root=str(staging_root)), - equipment=[ - EquipmentConfig( - id="EQ1", - label="Equipment 1", - local_root=str(staging_root), - nas_root="/nas", - completeness_signal=completeness_signal, - sentinel_filename=sentinel_filename - if completeness_signal == "sentinel_file" - else None, - manifest_filename=manifest_filename if completeness_signal == "manifest" else None, - transport=RcloneTransport( - type="rclone", - rclone_remote="lab-nas", - rclone_remote_path="/srv/nas", - bandwidth=BandwidthConfig(), - ), - ), - ], - orchestrator=OrchestratorConfig( - label="ORCH", - staging_root=str(staging_root), - staging_cleanup=OrchestratorStagingCleanup( - mode=cleanup_mode, retain_hours=retain_hours - ), - ), - ) - - -def _make_creation() -> CreationJson: - return msgspec.convert( - { - "schema_version": CREATION_JSON_VERSION, - "created_at": "2026-04-17T14:32:00Z", - "created_by": "asmith", - "level": "run", - "run_kind": "experimental", - "lims_project": { - "uid": "abc", - "short_id": "PROJ-0001", - "name_at_creation": "Test Project", - }, - "template": { - "name": "confocal_run", - "version": "1.0", - "source_path": "templates/confocal_run", - "run_scope": "experimental", - }, - "variables": {}, - "paths": { - "local": "/mnt/staging/EQ1/PROJ-0001/Run_2026-04-17T14-32-00", - "nas": "/nas/EQ1/PROJ-0001/Run_2026-04-17T14-32-00", - }, - }, - type=CreationJson, - ) - - -def _seed_pushed_run( - staging_root: Path, *, equipment: str = "EQ1", project: str = "PROJ-0001" -) -> Path: - """Create the on-disk push that an equipment machine would produce.""" - run_dir = staging_root / equipment / project / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - (run_dir / "data.bin").write_bytes(b"abcd" * 256) - cache = run_dir / CACHE_DIR_NAME - cache.mkdir() - creation = _make_creation() - (cache / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(creation)) - return run_dir - - -def _read_ingest(run_dir: Path) -> IngestJson: - return msgspec.json.decode( - (run_dir / CACHE_DIR_NAME / INGEST_JSON_NAME).read_bytes(), - type=IngestJson, - ) - - -# --------------------------------------------------------------------------- -# Bootstrap (no ingest.json yet -> writes the initial staging payload) -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_bootstraps_initial_ingest_json(tmp_path: Path) -> None: - config = _make_config(tmp_path) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - - state = await watcher.evaluate_run(run_dir) - - assert state == IngestState.STAGING - ingest = _read_ingest(run_dir) - assert ingest.current_state == IngestState.STAGING.value - assert ingest.equipment_id == "EQ1" - assert ingest.run_kind == "experimental" - assert ingest.transport == "smb_mount" - assert len(ingest.history) == 1 - assert ingest.history[0]["state"] == IngestState.STAGING.value - - -async def test_evaluate_run_bootstraps_when_creation_json_missing(tmp_path: Path) -> None: - """Equipment push that has no creation.json yet still gets a staging entry.""" - config = _make_config(tmp_path) - run_dir = tmp_path / "EQ1" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=None), - ) - - state = await watcher.evaluate_run(run_dir) - - assert state == IngestState.STAGING - ingest = _read_ingest(run_dir) - assert ingest.equipment_id == "EQ1" - - -async def test_evaluate_run_returns_staging_for_unrecognised_path(tmp_path: Path) -> None: - """A path whose leaf doesn't start with Run_ / TestRun_ is a no-op.""" - config = _make_config(tmp_path) - bogus = tmp_path / "EQ1" / "PROJ-0001" / "NotARun" - bogus.mkdir(parents=True) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(), - ) - state = await watcher.evaluate_run(bogus) - assert state == IngestState.STAGING - - -# --------------------------------------------------------------------------- -# staging -> complete (sentinel file) -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_advances_to_complete_when_sentinel_file_present(tmp_path: Path) -> None: - config = _make_config(tmp_path, sentinel_filename="run_complete") - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - # First call writes the staging entry. - await watcher.evaluate_run(run_dir) - # No sentinel yet -- still staging. - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.STAGING - - # Equipment writes the sentinel. - (run_dir / "run_complete").write_text("done") - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.COMPLETE - ingest = _read_ingest(run_dir) - last = ingest.history[-1] - assert last["state"] == IngestState.COMPLETE.value - assert last["files_received"] >= 1 - assert last["bytes_received"] >= 4 * 256 - - -# --------------------------------------------------------------------------- -# staging -> complete (manifest comparison) -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_advances_to_complete_when_manifest_satisfied(tmp_path: Path) -> None: - config = _make_config( - tmp_path, - completeness_signal="manifest", - sentinel_filename="", - manifest_filename="manifest.json", - ) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) # bootstrap - - # Write a manifest listing the existing data.bin with its actual size. - manifest = {"files": [{"path": "data.bin", "size": (run_dir / "data.bin").stat().st_size}]} - (run_dir / "manifest.json").write_bytes(msgspec.json.encode(manifest)) - - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.COMPLETE - - -async def test_evaluate_run_stays_staging_when_manifest_size_mismatches(tmp_path: Path) -> None: - config = _make_config( - tmp_path, - completeness_signal="manifest", - sentinel_filename="", - manifest_filename="manifest.json", - ) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - manifest = {"files": [{"path": "data.bin", "size": 999_999_999}]} - (run_dir / "manifest.json").write_bytes(msgspec.json.encode(manifest)) - - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.STAGING - - -async def test_evaluate_run_stays_staging_when_manifest_lists_missing_file(tmp_path: Path) -> None: - config = _make_config( - tmp_path, - completeness_signal="manifest", - sentinel_filename="", - manifest_filename="manifest.json", - ) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - manifest = {"files": [{"path": "missing.bin", "size": 100}]} - (run_dir / "manifest.json").write_bytes(msgspec.json.encode(manifest)) - - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.STAGING - - -# --------------------------------------------------------------------------- -# complete -> sync_queued -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_advances_to_sync_queued_after_enqueue(tmp_path: Path) -> None: - config = _make_config(tmp_path) - run_dir = _seed_pushed_run(tmp_path) - nas_sync = StubNasSync() - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=nas_sync, - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) # -> complete - - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.SYNC_QUEUED - assert nas_sync.enqueue_calls == [run_dir] - - -# --------------------------------------------------------------------------- -# sync_queued -> sync_verified -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_advances_to_sync_verified_when_status_verified(tmp_path: Path) -> None: - config = _make_config(tmp_path) - run_dir = _seed_pushed_run(tmp_path) - nas_sync = StubNasSync() - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=nas_sync, - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - await watcher.evaluate_run(run_dir) # -> sync_queued - - # Without the verified status, no transition. - nas_sync.status_responses[str(run_dir)] = "running" - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.SYNC_QUEUED - - # With verified status, advances. - nas_sync.status_responses[str(run_dir)] = "verified" - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.SYNC_VERIFIED - - -async def test_evaluate_run_treats_cleaned_status_as_verified(tmp_path: Path) -> None: - """Per §7.1.2 the cleanup states still mean the NAS copy is durable.""" - config = _make_config(tmp_path) - run_dir = _seed_pushed_run(tmp_path) - nas_sync = StubNasSync() - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=nas_sync, - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - await watcher.evaluate_run(run_dir) - nas_sync.status_responses[str(run_dir)] = "cleaned" - - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.SYNC_VERIFIED - - -# --------------------------------------------------------------------------- -# sync_verified -> cleared (manual + scheduled) -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_does_not_clear_in_manual_mode(tmp_path: Path) -> None: - config = _make_config(tmp_path, cleanup_mode=StagingCleanupMode.MANUAL.value) - run_dir = _seed_pushed_run(tmp_path) - nas_sync = StubNasSync() - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=nas_sync, - cache_creation=StubCreationCache(payload=_make_creation()), - ) - # Walk forward to sync_verified. - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - await watcher.evaluate_run(run_dir) - nas_sync.status_responses[str(run_dir)] = "verified" - await watcher.evaluate_run(run_dir) - - # Even after another tick, manual mode does not auto-clear. - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.SYNC_VERIFIED - assert run_dir.exists() - - -async def test_evaluate_run_clears_in_scheduled_mode_after_retain_hours(tmp_path: Path) -> None: - config = _make_config( - tmp_path, - cleanup_mode=StagingCleanupMode.SCHEDULED.value, - retain_hours=1, - ) - run_dir = _seed_pushed_run(tmp_path) - nas_sync = StubNasSync() - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=nas_sync, - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - await watcher.evaluate_run(run_dir) - nas_sync.status_responses[str(run_dir)] = "verified" - await watcher.evaluate_run(run_dir) - - # Patch the on-disk sync_verified entry to be 2 hours old. - cache_path = run_dir / CACHE_DIR_NAME / INGEST_JSON_NAME - payload = msgspec.json.decode(cache_path.read_bytes(), type=IngestJson) - backdated_at = (datetime.now(tz=UTC) - timedelta(hours=2)).strftime("%Y-%m-%dT%H:%M:%SZ") - new_history = [] - for entry in payload.history: - new_entry = dict(entry) - if entry.get("state") == IngestState.SYNC_VERIFIED.value: - new_entry["at"] = backdated_at - new_history.append(new_entry) - new_payload = msgspec.structs.replace(payload, history=new_history) - cache_path.write_bytes(msgspec.json.encode(new_payload)) - - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.CLEARED - assert not run_dir.exists() - - -# --------------------------------------------------------------------------- -# cleared is terminal -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_returns_cleared_when_already_cleared(tmp_path: Path) -> None: - """A run whose ingest says cleared but whose dir lingers is a no-op.""" - config = _make_config(tmp_path) - run_dir = _seed_pushed_run(tmp_path) - cache = run_dir / CACHE_DIR_NAME - payload = msgspec.convert( - { - "schema_version": INGEST_JSON_VERSION, - "project_name": "Test", - "equipment_id": "EQ1", - "run_kind": "experimental", - "run_path": str(run_dir), - "transport": "smb_mount", - "current_state": IngestState.CLEARED.value, - "history": [ - {"state": IngestState.CLEARED.value, "at": "2026-04-17T14:00:00Z", "host": "h"} - ], - }, - type=IngestJson, - ) - await IngestWriter().write_ingest(cache / INGEST_JSON_NAME, payload) - - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(), - ) - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.CLEARED - - -# --------------------------------------------------------------------------- -# poll_once + on_state_change hook -# --------------------------------------------------------------------------- - - -async def test_poll_once_returns_state_per_run(tmp_path: Path) -> None: - config = _make_config(tmp_path) - _seed_pushed_run(tmp_path, equipment="EQ1", project="PROJ-0001") - # Add a second equipment in the config + on disk. - config.equipment.append( - EquipmentConfig( - id="EQ2", - label="Equipment 2", - local_root=str(tmp_path), - nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="run_complete", - transport=RcloneTransport( - type="rclone", - rclone_remote="lab-nas", - rclone_remote_path="/srv/nas", - ), - ) - ) - run2 = tmp_path / "EQ2" / "PROJ-0002" / "Run_2026-04-18T00-00-00" - run2.mkdir(parents=True) - cache2 = run2 / CACHE_DIR_NAME - cache2.mkdir() - (cache2 / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(_make_creation())) - - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - states = await watcher.poll_once() - assert states == [IngestState.STAGING, IngestState.STAGING] - - -async def test_on_state_change_called_for_each_transition(tmp_path: Path) -> None: - config = _make_config(tmp_path) - run_dir = _seed_pushed_run(tmp_path) - captured: list[tuple[Path, IngestState]] = [] - - async def hook(path: Path, state: IngestState) -> None: - captured.append((path, state)) - - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - on_state_change=hook, - ) - await watcher.evaluate_run(run_dir) - (run_dir / "run_complete").write_text("done") - await watcher.evaluate_run(run_dir) - - states_seen = [s for (_, s) in captured] - assert IngestState.STAGING in states_seen - assert IngestState.COMPLETE in states_seen - - -def test_on_state_change_supports_sync_callable(tmp_path: Path) -> None: - """A non-async callback must also be invoked correctly.""" - config = _make_config(tmp_path) - captured: list[Any] = [] - - def hook(path: Path, state: IngestState) -> None: - captured.append((path, state)) - - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - on_state_change=hook, - ) - - async def driver() -> None: - run_dir = _seed_pushed_run(tmp_path) - await watcher.evaluate_run(run_dir) - - asyncio.run(driver()) - assert captured # at least the staging transition was recorded - - -# --------------------------------------------------------------------------- -# start / stop lifecycle -# --------------------------------------------------------------------------- - - -async def test_start_with_missing_staging_root_is_no_op(tmp_path: Path) -> None: - """Redesign §3.1: orchestrator pipeline is always active. A missing - staging_root on disk is handled per-poll as a no-op, not by skipping - the start() call.""" - config = _make_config(tmp_path / "does-not-exist") - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(), - ) - await watcher.start() - # Poll once with a non-existent staging root returns empty. - states = await watcher.poll_once() - assert states == [] - await watcher.stop() - - -async def test_start_then_stop_runs_the_loop(tmp_path: Path) -> None: - config = _make_config(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(), - poll_interval_s=0.01, - ) - await watcher.start() - # Give the loop a moment to tick at least once. - await asyncio.sleep(0.05) - await watcher.stop() - - -async def test_start_is_idempotent(tmp_path: Path) -> None: - config = _make_config(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(), - poll_interval_s=0.01, - ) - await watcher.start() - await watcher.start() # second call must be a no-op - await watcher.stop() - - -# --------------------------------------------------------------------------- -# Edge case: equipment without orchestrator_staging_transport -# --------------------------------------------------------------------------- - - -async def test_evaluate_run_uses_default_transport_when_unset(tmp_path: Path) -> None: - config = _make_config(tmp_path) - config.equipment[0] = EquipmentConfig( - id="EQ1", - label="Equipment 1", - local_root=str(tmp_path), - nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="run_complete", - transport=RcloneTransport( - type="rclone", - rclone_remote="lab-nas", - rclone_remote_path="/srv/nas", - ), - # No orchestrator_staging_transport. - ) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - ingest = _read_ingest(run_dir) - assert ingest.transport == "smb_mount" - - -async def test_evaluate_run_handles_path_outside_staging_root(tmp_path: Path) -> None: - """A run whose absolute path is not inside the staging tree is a no-op.""" - config = _make_config(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(), - ) - outside = tmp_path.parent / "outside" / "Run_2026-04-17T14-32-00" - outside.mkdir(parents=True, exist_ok=True) - state = await watcher.evaluate_run(outside) - assert state == IngestState.STAGING - - -async def test_evaluate_run_advances_when_manifest_with_no_files_present(tmp_path: Path) -> None: - """An empty-files manifest means 'no files expected'; sentinel alone is enough.""" - config = _make_config( - tmp_path, - completeness_signal="manifest", - sentinel_filename="", - manifest_filename="manifest.json", - ) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - (run_dir / "manifest.json").write_bytes(msgspec.json.encode({"files": []})) - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.COMPLETE - - -async def test_evaluate_run_stays_staging_when_manifest_malformed(tmp_path: Path) -> None: - config = _make_config( - tmp_path, - completeness_signal="manifest", - sentinel_filename="", - manifest_filename="manifest.json", - ) - run_dir = _seed_pushed_run(tmp_path) - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - (run_dir / "manifest.json").write_bytes(b"not json {") - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.STAGING - - -async def test_completeness_signal_unknown_equipment_returns_false(tmp_path: Path) -> None: - """A run whose equipment id isn't configured stays staging.""" - config = _make_config(tmp_path) - run_dir = tmp_path / "UNKNOWN" / "PROJ-0001" / "Run_2026-04-17T14-32-00" - run_dir.mkdir(parents=True) - cache = run_dir / CACHE_DIR_NAME - cache.mkdir() - (cache / CREATION_JSON_NAME).write_bytes(msgspec.json.encode(_make_creation())) - - watcher = StagingWatcher( - config=config, - ingest_writer=IngestWriter(), - nas_sync=StubNasSync(), - cache_creation=StubCreationCache(payload=_make_creation()), - ) - await watcher.evaluate_run(run_dir) - # Even with a sentinel present, an unknown equipment never advances. - (run_dir / "run_complete").write_text("done") - state = await watcher.evaluate_run(run_dir) - assert state == IngestState.STAGING diff --git a/tests/unit/sync/test_nas_client.py b/tests/unit/sync/test_nas_client.py index aa685f1..ea568cb 100644 --- a/tests/unit/sync/test_nas_client.py +++ b/tests/unit/sync/test_nas_client.py @@ -65,8 +65,6 @@ def _build_config(local_root: Path, *, retain_cache: bool = True) -> Config: label="Eq 1", local_root=str(local_root), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="DONE", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -122,7 +120,9 @@ def _make_push_factory( ) -> Callable[[EquipmentConfig], Callable[..., Any]]: """A push callable factory that yields deterministic outcomes for tests.""" - async def _push(local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=ok, error_kind=error_kind, returncode=0 if ok else 1) def factory(_eq: EquipmentConfig) -> Callable[..., Any]: @@ -439,7 +439,9 @@ async def test_enqueue_idempotent_for_already_queued_row( run_dir = await _populate_run(tmp_path) # Use a slow stub to keep the row from progressing past QUEUED. - async def _slow(local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _slow( + local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: await asyncio.sleep(0.5) return TransportResult(ok=True) @@ -523,3 +525,103 @@ async def test_mark_cleaned_is_noop_when_creation_json_missing( await client._mark_cleaned(run_dir) finally: await client.close() + + +# --------------------------------------------------------------------------- +# Phase 4: per-file enqueue + verify reconciliation +# --------------------------------------------------------------------------- + + +async def test_enqueue_with_files_inserts_subset(tmp_path: Path, writer: CreationWriter) -> None: + """``enqueue(run, files=[...])`` stores the subset on the queue row.""" + cfg = _build_config(tmp_path) + run_dir = await _populate_run(tmp_path) + + async def _slow(local: Path, *, bwlimit_kibps: int | None, files_from: object = None): + await asyncio.sleep(0.5) + return TransportResult(ok=True) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + push_callable_factory=lambda _eq: _slow, + worker_poll_interval_s=0.01, + ) + await client.init() + try: + handle = await client.enqueue(run_dir, ["data.bin"]) + assert handle.state == HandleState.QUEUED + row = await client._queue.get_by_run_path(run_dir) + assert row is not None + assert row.files == ("data.bin",) + finally: + await client.close() + + +async def test_enqueue_requeues_terminal_job_with_new_files( + tmp_path: Path, writer: CreationWriter +) -> None: + """A terminal (FAILED) job is re-armed in QUEUED with a fresh file subset.""" + cfg = _build_config(tmp_path) + run_dir = await _populate_run(tmp_path) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + push_callable_factory=_make_push_factory(ok=False, error_kind=TransportErrorKind.AUTH), + worker_poll_interval_s=0.01, + ) + await client.init() + try: + handle = await client.enqueue(run_dir, ["data.bin"]) + for _ in range(200): + row = await client._queue.get_by_id(handle.job_id) + if row is not None and row.state is SyncJobState.FAILED: + break + await asyncio.sleep(0.02) + else: + pytest.fail("worker did not reach FAILED") + # Re-enqueue with a new subset -> re-armed QUEUED carrying the new files. + handle2 = await client.enqueue(run_dir, ["other.bin"]) + assert handle2.state == HandleState.QUEUED + row2 = await client._queue.get_by_run_path(run_dir) + assert row2 is not None + assert row2.files == ("other.bin",) + finally: + await client.close() + + +async def test_enqueue_noops_active_job_with_new_files( + tmp_path: Path, writer: CreationWriter +) -> None: + """An active (QUEUED) job is left untouched when re-enqueued with new files.""" + cfg = _build_config(tmp_path) + run_dir = await _populate_run(tmp_path) + + async def _slow(local: Path, *, bwlimit_kibps: int | None, files_from: object = None): + await asyncio.sleep(0.5) + return TransportResult(ok=True) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + push_callable_factory=lambda _eq: _slow, + worker_poll_interval_s=0.01, + ) + await client.init() + try: + handle1 = await client.enqueue(run_dir, ["data.bin"]) + handle2 = await client.enqueue(run_dir, ["other.bin"]) + # Same job; the new files do NOT overwrite the active row's subset. + assert handle1.job_id == handle2.job_id + row = await client._queue.get_by_run_path(run_dir) + assert row is not None + assert row.files == ("data.bin",) + finally: + await client.close() diff --git a/tests/unit/sync/test_nas_client_extra.py b/tests/unit/sync/test_nas_client_extra.py index ad148b2..c5702fa 100644 --- a/tests/unit/sync/test_nas_client_extra.py +++ b/tests/unit/sync/test_nas_client_extra.py @@ -71,8 +71,6 @@ def _build_config( label="Eq 1", local_root=str(local_root), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="DONE", transport=transport, ) ], @@ -128,8 +126,6 @@ def test_build_transport_driver_rclone(tmp_path: Path) -> None: label="Eq 1", local_root=str(tmp_path), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="DONE", transport=RcloneTransport( type="rclone", rclone_remote="lab-nas", @@ -148,8 +144,6 @@ def test_build_transport_driver_rsync_ssh(tmp_path: Path) -> None: label="Eq 1", local_root=str(tmp_path), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="DONE", transport=RsyncSshTransport( type="rsync_ssh", ssh_target="user@host", @@ -180,7 +174,9 @@ async def test_hash_mismatch_first_failure_retries(tmp_path: Path) -> None: writer = CreationWriter(lock_timeout_seconds=10.0) call_count = {"n": 0} - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: call_count["n"] += 1 if call_count["n"] == 1: return TransportResult( @@ -229,7 +225,9 @@ async def test_hash_mismatch_second_failure_terminal(tmp_path: Path) -> None: run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=False, error_kind=TransportErrorKind.HASH_MISMATCH, returncode=1) client = NASSyncClient( @@ -265,7 +263,9 @@ async def test_cleanup_full_delete_when_retain_cache_false(tmp_path: Path) -> No run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -299,7 +299,9 @@ async def test_cleanup_retain_cache_keeps_metadata(tmp_path: Path) -> None: run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -335,7 +337,9 @@ async def test_cleanup_disabled_keeps_files(tmp_path: Path) -> None: run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -371,7 +375,9 @@ async def test_cleanup_eligible_when_min_verify_passes_unmet(tmp_path: Path) -> run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -405,7 +411,9 @@ async def test_cleanup_blocked_by_remote_stat(tmp_path: Path) -> None: run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -434,6 +442,275 @@ async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: await client.close() +# --------------------------------------------------------------------------- +# Phase 5: _delete_local honors keep_local; cleanup gates on the SYNCED rollup +# --------------------------------------------------------------------------- + + +def _client(cfg: Config, tmp_path: Path) -> NASSyncClient: + """Build an un-init'd client for direct ``_delete_local`` calls.""" + return NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=CreationWriter(lock_timeout_seconds=10.0), + ) + + +def test_delete_local_skips_keep_local_files_incl_nested(tmp_path: Path) -> None: + """``_delete_local`` keeps ``keep_local`` files, incl. a nested one.""" + run_dir = tmp_path / "EQ1" / "PROJ-0042" / "Runs" / "Run_x" + run_dir.mkdir(parents=True) + (run_dir / "keep.bin").write_bytes(b"keep") + (run_dir / "drop.bin").write_bytes(b"drop") + nested = run_dir / "sub" / "deep" + nested.mkdir(parents=True) + (nested / "kept_nested.txt").write_text("nested-keep") + (run_dir / "sub" / "other.txt").write_text("nested-drop") + cache = run_dir / CACHE_DIR_NAME + cache.mkdir() + (cache / "meta.json").write_text("{}") + + cfg = _build_config(tmp_path, retain_cache=True) + client = _client(cfg, tmp_path) + client._delete_local(run_dir, {"keep.bin", "sub/deep/kept_nested.txt"}) + + # Retained files survive; the run dir + cache are intact. + assert (run_dir / "keep.bin").exists() + assert (nested / "kept_nested.txt").exists() + assert (cache / "meta.json").exists() + # Non-retained files are gone. + assert not (run_dir / "drop.bin").exists() + assert not (run_dir / "sub" / "other.txt").exists() + # The directory holding the kept nested file survives. + assert (run_dir / "sub" / "deep").exists() + + +def test_delete_local_prunes_emptied_dirs_keeps_dir_with_kept_file(tmp_path: Path) -> None: + """A directory holding only deleted files is pruned (deepest-first); + a parent still holding a kept file survives.""" + run_dir = tmp_path / "EQ1" / "PROJ-0042" / "Runs" / "Run_p" + run_dir.mkdir(parents=True) + # ``branch/`` keeps a file directly + has a fully-emptied descendant. + branch = run_dir / "branch" + (branch / "leaf").mkdir(parents=True) + (branch / "kept.txt").write_text("keep") + (branch / "leaf" / "drop_a.txt").write_text("a") + (branch / "leaf" / "drop_b.txt").write_text("b") + # ``gone/`` and its nested ``gone/inner/`` hold only droppable files -> + # the whole ``gone`` subtree must be pruned away (deepest-first). + inner = run_dir / "gone" / "inner" + inner.mkdir(parents=True) + (run_dir / "gone" / "drop_c.txt").write_text("c") + (inner / "drop_d.txt").write_text("d") + + cfg = _build_config(tmp_path, retain_cache=True) + client = _client(cfg, tmp_path) + client._delete_local(run_dir, {"branch/kept.txt"}) + + # The kept file and its directory survive. + assert (branch / "kept.txt").exists() + assert branch.is_dir() + # The fully-emptied descendant directory is pruned. + assert not (branch / "leaf").exists() + # The entire ``gone`` subtree (parent + nested) is pruned deepest-first. + assert not (run_dir / "gone").exists() + # The run directory itself is never removed. + assert run_dir.is_dir() + + +def test_delete_local_does_not_descend_or_remove_directory_symlink(tmp_path: Path) -> None: + """A directory symlink inside the run is left untouched -- its target's + contents are not deleted and the link itself is not removed.""" + # An external directory the run will symlink to. + external = tmp_path / "external" + external.mkdir() + (external / "precious.txt").write_text("do-not-delete") + + run_dir = tmp_path / "EQ1" / "PROJ-0042" / "Runs" / "Run_s" + run_dir.mkdir(parents=True) + (run_dir / "drop.bin").write_bytes(b"drop") + link = run_dir / "linked" + link.symlink_to(external, target_is_directory=True) + + cfg = _build_config(tmp_path, retain_cache=True) + client = _client(cfg, tmp_path) + client._delete_local(run_dir, set()) + + # The run's own file is gone. + assert not (run_dir / "drop.bin").exists() + # The symlink itself and the external target's contents are untouched. + assert link.is_symlink() + assert external.is_dir() + assert (external / "precious.txt").read_text() == "do-not-delete" + + +def test_delete_local_keep_local_survives_retain_cache_false(tmp_path: Path) -> None: + """A ``keep_local`` file survives even when ``retain_cache=False``.""" + run_dir = tmp_path / "EQ1" / "PROJ-0042" / "Runs" / "Run_y" + run_dir.mkdir(parents=True) + (run_dir / "keep.bin").write_bytes(b"keep") + (run_dir / "drop.bin").write_bytes(b"drop") + + cfg = _build_config(tmp_path, retain_cache=False) + client = _client(cfg, tmp_path) + client._delete_local(run_dir, {"keep.bin"}) + + # The whole-run rmtree is skipped because a keep_local file exists. + assert run_dir.exists() + assert (run_dir / "keep.bin").exists() + assert not (run_dir / "drop.bin").exists() + + +def test_delete_local_retain_cache_false_drops_whole_run_without_keep_local( + tmp_path: Path, +) -> None: + """With ``retain_cache=False`` and no kept files the run dir is removed.""" + run_dir = tmp_path / "EQ1" / "PROJ-0042" / "Runs" / "Run_z" + run_dir.mkdir(parents=True) + (run_dir / "drop.bin").write_bytes(b"drop") + + cfg = _build_config(tmp_path, retain_cache=False) + client = _client(cfg, tmp_path) + client._delete_local(run_dir, set()) + assert not run_dir.exists() + + +async def test_cleanup_marks_cleared_in_sync_state(tmp_path: Path) -> None: + """A full cleanup pass stamps ``cleared_at`` in ``sync_state.json``.""" + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + cfg = _build_config(tmp_path, retain_cache=True, min_verify_passes=1, min_age_hours=0) + run_dir = await _populate_run(tmp_path) + writer = CreationWriter(lock_timeout_seconds=10.0) + + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: + return TransportResult(ok=True, returncode=0) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + push_callable_factory=_factory(_push), + hashsum_callable_factory=local_hashsum_factory(), + worker_poll_interval_s=0.005, + ) + await client.init() + try: + handle = await client.enqueue(run_dir) + for _ in range(400): + row = await client._queue.get_by_id(handle.job_id) + if row is not None and row.state is SyncJobState.CLEANED: + break + await asyncio.sleep(0.01) + else: + pytest.fail("expected CLEANED state") + finally: + await client.close() + + state = await SyncStateWriter().read(run_dir) + assert state.cleared_at is not None + assert SyncStateWriter.rollup_state(state).value == "cleared" + + +async def test_cleanup_keeps_keep_local_file(tmp_path: Path) -> None: + """A file flagged ``keep_local`` survives the cleanup sweep.""" + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + cfg = _build_config(tmp_path, retain_cache=True, min_verify_passes=1, min_age_hours=0) + run_dir = await _populate_run(tmp_path) + writer = CreationWriter(lock_timeout_seconds=10.0) + sync_writer = SyncStateWriter() + # Operator flags the top-level data file as keep-local before cleanup. + await sync_writer.set_keep_local(run_dir, "data.bin", True) + + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: + return TransportResult(ok=True, returncode=0) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + sync_state_writer=sync_writer, + push_callable_factory=_factory(_push), + hashsum_callable_factory=local_hashsum_factory(), + worker_poll_interval_s=0.005, + ) + await client.init() + try: + handle = await client.enqueue(run_dir) + for _ in range(400): + row = await client._queue.get_by_id(handle.job_id) + if row is not None and row.state is SyncJobState.CLEANED: + break + await asyncio.sleep(0.01) + else: + pytest.fail("expected CLEANED state") + # The keep_local file survives; the other data file is removed. + assert (run_dir / "data.bin").exists() + assert not (run_dir / "subdir" / "child.txt").exists() + finally: + await client.close() + + +async def test_cleanup_deferred_when_run_only_partially_synced(tmp_path: Path) -> None: + """Cleanup does not run while a tracked file remains unverified. + + A pre-existing ``sync_state.json`` records an extra file that never + verifies, so the whole-run rollup stays ``SYNCING`` even after this + job's subset verifies -- the job promotes to VERIFIED but cleanup is + deferred and the local files survive. + """ + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + cfg = _build_config(tmp_path, retain_cache=True, min_verify_passes=1, min_age_hours=0) + run_dir = await _populate_run(tmp_path) + writer = CreationWriter(lock_timeout_seconds=10.0) + sync_writer = SyncStateWriter() + # A later-sweep file that has never verified -> run is not fully SYNCED. + await sync_writer.upsert_file(run_dir, "pending.bin", synced_signature=None, verified_at=None) + + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: + return TransportResult(ok=True, returncode=0) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + sync_state_writer=sync_writer, + push_callable_factory=_factory(_push), + hashsum_callable_factory=local_hashsum_factory(), + worker_poll_interval_s=0.005, + ) + await client.init() + try: + handle = await client.enqueue(run_dir, files=["data.bin"]) + for _ in range(400): + row = await client._queue.get_by_id(handle.job_id) + if row is not None and row.state is SyncJobState.VERIFIED: + break + await asyncio.sleep(0.01) + else: + pytest.fail("expected VERIFIED") + # Give the worker a beat -- cleanup must NOT advance the job. + await asyncio.sleep(0.1) + row = await client._queue.get_by_id(handle.job_id) + assert row is not None and row.state is SyncJobState.VERIFIED + # Local data is retained because the run is not fully SYNCED. + assert (run_dir / "data.bin").exists() + finally: + await client.close() + + # --------------------------------------------------------------------------- # Worker error handling: equipment-not-configured / vanished local # --------------------------------------------------------------------------- @@ -448,7 +725,9 @@ async def test_worker_marks_failed_when_local_run_vanished(tmp_path: Path) -> No # Use a slow stub so we have time to delete the directory before the # worker picks the row. - async def _slow(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _slow( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: await asyncio.sleep(2.0) return TransportResult(ok=True) @@ -497,8 +776,10 @@ class _StubVerifier(Verifier): def __init__(self) -> None: self.calls = 0 - async def compute_local_manifest(self, run_path: Path) -> dict[str, str]: - return await Verifier.compute_local_manifest(self, run_path) + async def compute_local_manifest( + self, run_path: Path, include: set[str] | None = None + ) -> dict[str, str]: + return await Verifier.compute_local_manifest(self, run_path, include) async def verify_against_local( self, run_path: Path, manifest: dict[str, str] @@ -508,7 +789,9 @@ async def verify_against_local( return VerifyResult(ok=False, mismatched=("data.bin",), manifest=manifest) return VerifyResult(ok=True, manifest=manifest) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -553,7 +836,9 @@ async def verify_against_local( ) -> VerifyResult: return VerifyResult(ok=False, mismatched=("x",), manifest=manifest) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=True, returncode=0) client = NASSyncClient( @@ -608,7 +893,9 @@ async def test_network_error_records_backoff_retry(tmp_path: Path) -> None: run_dir = await _populate_run(tmp_path) writer = CreationWriter(lock_timeout_seconds=10.0) - async def _push(_local: Path, *, bwlimit_kibps: int | None) -> TransportResult: + async def _push( + _local: Path, *, bwlimit_kibps: int | None, files_from: object = None + ) -> TransportResult: return TransportResult(ok=False, error_kind=TransportErrorKind.NETWORK, returncode=1) client = NASSyncClient( @@ -663,8 +950,6 @@ class _BogusTransport: label="Eq", local_root=str(tmp_path), nas_root="/nas", - completeness_signal="sentinel_file", - sentinel_filename="DONE", transport=_BogusTransport(), # type: ignore[arg-type] ) with pytest.raises(ValueError, match="unsupported transport"): @@ -727,3 +1012,125 @@ def test_infer_equipment_id_falls_back_to_first(tmp_path: Path) -> None: ) inferred = client._infer_equipment_id(Path("/no/match/here"), creation) assert inferred == "EQ1" + + +# --------------------------------------------------------------------------- +# Phase 4: per-file verify reconciliation +# --------------------------------------------------------------------------- + + +async def test_partial_batch_credits_verified_files_in_sync_state(tmp_path: Path) -> None: + """A batch where one file fails verification still credits the others + in ``sync_state.json`` -- operator-free per-file NAS sync 'Failure + handling': a single bad file must not block the good ones.""" + import hashlib + + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + cfg = _build_config(tmp_path) + run_dir = await _populate_run(tmp_path) # data.bin + subdir/child.txt + writer = CreationWriter(lock_timeout_seconds=10.0) + sync_state = SyncStateWriter() + + async def _push(_local: Path, *, bwlimit_kibps: int | None, files_from: object = None): + return TransportResult(ok=True, returncode=0) + + def corrupt_one_hashsum_factory(): + """Remote-hash probe that mangles ``subdir/child.txt`` -- so it + mismatches while ``data.bin`` verifies cleanly.""" + + async def _hashsum(target: Path) -> dict[str, str]: + out: dict[str, str] = {} + for f in sorted(target.rglob("*")): + if not f.is_file(): + continue + rel = f.relative_to(target).as_posix() + if rel.startswith(".exlab-wizard/"): + continue + digest = hashlib.sha256(f.read_bytes()).hexdigest() + if rel == "subdir/child.txt": + digest = "0" * 64 # corrupt this one file's remote digest + out[rel] = digest + return out + + return lambda _eq: _hashsum + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + sync_state_writer=sync_state, + push_callable_factory=_factory(_push), + hashsum_callable_factory=corrupt_one_hashsum_factory(), + worker_poll_interval_s=0.005, + ) + await client.init() + try: + handle = await client.enqueue(run_dir, ["data.bin", "subdir/child.txt"]) + # child.txt mismatches on every probe -> the hash-mismatch path + # retries once then terminates the whole batch job at FAILED. + for _ in range(600): + row = await client._queue.get_by_id(handle.job_id) + if row is not None and row.state is SyncJobState.FAILED: + break + await asyncio.sleep(0.01) + else: + pytest.fail("expected the batch job to terminate FAILED") + + # Even though the batch job is terminal FAILED, per-file + # reconciliation must still have credited data.bin: it verified, + # so it carries a synced_signature + verified_at. child.txt did + # NOT verify -> uncredited. + state = await sync_state.read(run_dir) + assert "data.bin" in state.files + assert state.files["data.bin"].synced_signature is not None + assert state.files["data.bin"].verified_at is not None + assert "subdir/child.txt" not in state.files + finally: + await client.close() + + +async def test_full_batch_credits_every_file_in_sync_state(tmp_path: Path) -> None: + """A fully-successful batch credits every verified file in sync_state.json.""" + from exlab_wizard.cache.sync_state_writer import SyncStateWriter + + cfg = _build_config(tmp_path) + run_dir = await _populate_run(tmp_path) + writer = CreationWriter(lock_timeout_seconds=10.0) + sync_state = SyncStateWriter() + + async def _push(_local: Path, *, bwlimit_kibps: int | None, files_from: object = None): + return TransportResult(ok=True, returncode=0) + + client = NASSyncClient( + config=cfg, + queue_db=tmp_path / "q.db", + validator=Validator(), + cache_creation=writer, + sync_state_writer=sync_state, + push_callable_factory=_factory(_push), + hashsum_callable_factory=local_hashsum_factory(), + worker_poll_interval_s=0.005, + ) + await client.init() + try: + handle = await client.enqueue(run_dir, ["data.bin", "subdir/child.txt"]) + for _ in range(400): + row = await client._queue.get_by_id(handle.job_id) + if row is not None and row.state in { + SyncJobState.VERIFIED, + SyncJobState.CLEANUP_ELIGIBLE, + SyncJobState.CLEANED, + }: + break + await asyncio.sleep(0.01) + else: + pytest.fail("expected eventual VERIFIED") + state = await sync_state.read(run_dir) + assert {"data.bin", "subdir/child.txt"} <= set(state.files) + for rec in state.files.values(): + assert rec.synced_signature is not None + assert rec.verified_at is not None + finally: + await client.close() diff --git a/tests/unit/sync/test_queue.py b/tests/unit/sync/test_queue.py index 606ab1a..a08e563 100644 --- a/tests/unit/sync/test_queue.py +++ b/tests/unit/sync/test_queue.py @@ -49,6 +49,37 @@ async def test_insert_unique_constraint(queue: SyncQueue, tmp_path: Path) -> Non await queue.insert(run_path=tmp_path / "run", equipment_id="EQ1") +async def test_files_column_round_trips(queue: SyncQueue, tmp_path: Path) -> None: + """The per-file ``files`` list round-trips through insert and read.""" + files = ["data.bin", "subdir/child.txt"] + row = await queue.insert(run_path=tmp_path / "run", equipment_id="EQ1", files=files) + assert row.files == tuple(files) + read_back = await queue.get_by_run_path(tmp_path / "run") + assert read_back is not None + assert read_back.files == tuple(files) + + +async def test_files_column_defaults_to_empty(queue: SyncQueue, tmp_path: Path) -> None: + """A job inserted without ``files`` reads back as the empty 'whole run' tuple.""" + row = await queue.insert(run_path=tmp_path / "run", equipment_id="EQ1") + assert row.files == () + read_back = await queue.get_by_run_path(tmp_path / "run") + assert read_back is not None + assert read_back.files == () + + +async def test_requeue_with_files_resets_terminal_job(queue: SyncQueue, tmp_path: Path) -> None: + """``requeue_with_files`` re-arms a terminal job in QUEUED with a new subset.""" + row = await queue.insert(run_path=tmp_path / "run", equipment_id="EQ1", files=["old.bin"]) + await queue.transition(row.id, SyncJobState.VERIFIED, increment_verify_passes=True) + requeued = await queue.requeue_with_files(row.id, ["new.bin", "another.bin"]) + assert requeued.state is SyncJobState.QUEUED + assert requeued.files == ("new.bin", "another.bin") + assert requeued.attempts == 0 + assert requeued.verify_passes == 0 + assert requeued.verified_at is None + + async def test_get_by_id_and_run_path(queue: SyncQueue, tmp_path: Path) -> None: row = await queue.insert(run_path=tmp_path / "run", equipment_id="EQ1") by_id = await queue.get_by_id(row.id) diff --git a/tests/unit/sync/test_transports.py b/tests/unit/sync/test_transports.py index ffc1cb9..f1d3088 100644 --- a/tests/unit/sync/test_transports.py +++ b/tests/unit/sync/test_transports.py @@ -368,6 +368,80 @@ async def test_rclone_argv_omits_bwlimit_when_none( assert "--bwlimit" not in argv +async def test_rclone_argv_includes_files_from_when_set( + stub_dir: Path, + record_argv: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``files_from`` -> ``--files-from`` immediately followed by the path.""" + src = tmp_path / "src" + src.mkdir() + files_from = tmp_path / "files.txt" + files_from.write_text("data.bin\n") + monkeypatch.setenv("STUB_RCLONE_BEHAVIOR", "success") + transport = RcloneTransport() + await transport.push(src, "remote:/srv/run", files_from=files_from) + argv = _read_recorded_argvs(record_argv)[0] + assert "--files-from" in argv + idx = argv.index("--files-from") + assert argv[idx + 1] == str(files_from) + + +async def test_rclone_argv_omits_files_from_when_none( + stub_dir: Path, + record_argv: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``files_from=None`` -> ``--files-from`` is absent from argv.""" + src = tmp_path / "src" + src.mkdir() + monkeypatch.setenv("STUB_RCLONE_BEHAVIOR", "success") + transport = RcloneTransport() + await transport.push(src, "remote:/srv/run") + argv = _read_recorded_argvs(record_argv)[0] + assert "--files-from" not in argv + + +async def test_rsync_argv_includes_files_from_when_set( + stub_dir: Path, + record_argv: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``files_from`` -> ``--files-from=`` is present in the rsync argv.""" + src = tmp_path / "src" + src.mkdir() + files_from = tmp_path / "files.txt" + files_from.write_text("data.bin\n") + monkeypatch.setenv("STUB_RSYNC_BEHAVIOR", "success") + transport = RsyncSshTransport() + key = tmp_path / "id_ed25519" + key.write_bytes(b"k") + await transport.push(src, "user@host", key, "/srv/run", files_from=files_from) + argv = _read_recorded_argvs(record_argv)[0] + assert f"--files-from={files_from}" in argv + + +async def test_rsync_argv_omits_files_from_when_none( + stub_dir: Path, + record_argv: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``files_from=None`` -> no ``--files-from`` flag in the rsync argv.""" + src = tmp_path / "src" + src.mkdir() + monkeypatch.setenv("STUB_RSYNC_BEHAVIOR", "success") + transport = RsyncSshTransport() + key = tmp_path / "id_ed25519" + key.write_bytes(b"k") + await transport.push(src, "user@host", key, "/srv/run") + argv = _read_recorded_argvs(record_argv)[0] + assert not any(a.startswith("--files-from") for a in argv) + + async def test_rsync_argv_includes_checksum_partial( stub_dir: Path, record_argv: Path, diff --git a/tests/unit/sync/test_verifier.py b/tests/unit/sync/test_verifier.py index ab09153..c3f987e 100644 --- a/tests/unit/sync/test_verifier.py +++ b/tests/unit/sync/test_verifier.py @@ -61,6 +61,44 @@ async def test_compute_local_manifest_writes_checksums_file(tmp_path: Path) -> N assert parsed == manifest +async def test_compute_local_manifest_include_filters_to_subset(tmp_path: Path) -> None: + """``include`` scopes the manifest to the given run-relative subset.""" + run_path = tmp_path / "run" + run_path.mkdir() + contents = _populate_run(run_path) + + verifier = Verifier() + manifest = await verifier.compute_local_manifest(run_path, {"data/a.txt", "metadata.json"}) + + assert set(manifest) == {"data/a.txt", "metadata.json"} + assert "data/b.txt" not in manifest + assert manifest["data/a.txt"] == hashlib.sha256(contents["data/a.txt"]).hexdigest() + + +async def test_compute_local_manifest_include_empty_set_hashes_nothing(tmp_path: Path) -> None: + """An empty ``include`` set yields an empty manifest (whole-run is ``None``).""" + run_path = tmp_path / "run" + run_path.mkdir() + _populate_run(run_path) + + manifest = await Verifier().compute_local_manifest(run_path, set()) + assert manifest == {} + + +async def test_compute_local_manifest_subset_skips_checksums_side_effect(tmp_path: Path) -> None: + """A subset (``include``) pass does NOT write ``checksums.sha256``. + + Persisting a partial manifest would clobber the run's durable checksum + file with an incomplete record; only a whole-run pass is durable. + """ + run_path = tmp_path / "run" + run_path.mkdir() + _populate_run(run_path) + + await Verifier().compute_local_manifest(run_path, {"data/a.txt"}) + assert not (run_path / CHECKSUMS_RELATIVE).exists() + + async def test_compute_excludes_cache_dir(tmp_path: Path) -> None: """Files under ``.exlab-wizard/`` are excluded from the manifest.""" run_path = tmp_path / "run" diff --git a/tests/unit/test_paths.py b/tests/unit/test_paths.py index 53ad9b5..6437677 100644 --- a/tests/unit/test_paths.py +++ b/tests/unit/test_paths.py @@ -57,8 +57,6 @@ def _make_equipment(equipment_id: str = "CONFOCAL_01") -> EquipmentConfig: "label": "Confocal Microscope", "local_root": "/data/lab", "nas_root": "//nas01/lab", - "completeness_signal": "sentinel_file", - "sentinel_filename": "done.flag", "transport": { "type": "rclone", "rclone_remote": "lab-nas", diff --git a/tests/unit/tray/test_dependencies.py b/tests/unit/tray/test_dependencies.py index 89a2991..a26308f 100644 --- a/tests/unit/tray/test_dependencies.py +++ b/tests/unit/tray/test_dependencies.py @@ -17,10 +17,19 @@ import keyring import keyring.backend - -from exlab_wizard.config.models import Config -from exlab_wizard.constants import KEYRING_USERNAME_LIMS +import pytest + +from exlab_wizard.config.models import ( + BandwidthConfig, + Config, + EquipmentConfig, + PathsConfig, + RcloneTransport, +) +from exlab_wizard.constants import KEYRING_USERNAME_LIMS, SyncMode from exlab_wizard.lims.keyring_store import KeyringStore +from exlab_wizard.sync.nas_client import NASSyncClient +from exlab_wizard.tray import dependencies as deps_module from exlab_wizard.tray.dependencies import ( _build_lims_client, _check_keyring_present, @@ -64,6 +73,45 @@ def test_build_production_dependencies_exposes_keyring_store(tmp_path: Path) -> assert isinstance(deps.keyring_store, KeyringStore) +def test_build_production_dependencies_nas_sync_is_a_client( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``deps.nas_sync`` must be a :class:`NASSyncClient`, not a bare queue. + + Regression: ``_build_nas_sync`` once returned a ``SyncQueue``, which + has neither ``enqueue`` nor ``status`` -- the poller sweep and the + force-sync route call both, so the bug surfaced only as a silent dead + sync loop in production. This is the test that catches it. + """ + local_root = tmp_path / "lab-data" + local_root.mkdir() + config = Config( + paths=PathsConfig(local_root=str(local_root)), + equipment=[ + EquipmentConfig( + id="EQNAS", + label="Nas Equipment", + local_root=str(local_root), + nas_root="/nas", + sync_mode=SyncMode.NAS, + transport=RcloneTransport( + type="rclone", + rclone_remote="lab-nas", + rclone_remote_path="/srv/nas", + bandwidth=BandwidthConfig(), + ), + ), + ], + ) + monkeypatch.setattr(deps_module, "_load_config_safely", lambda: config) + + deps = build_production_dependencies(tmp_path) + + assert isinstance(deps.nas_sync, NASSyncClient) + assert callable(deps.nas_sync.enqueue) + assert callable(deps.nas_sync.status) + + def test_check_keyring_present_true_when_lims_password_stored(tmp_path: Path) -> None: """The probe must look up the password under KEYRING_USERNAME_LIMS. diff --git a/tests/unit/ui/test_components.py b/tests/unit/ui/test_components.py index 91b75db..5fd87fc 100644 --- a/tests/unit/ui/test_components.py +++ b/tests/unit/ui/test_components.py @@ -133,6 +133,33 @@ def test_sync_status_cleaned_uses_success_with_cloud_icon() -> None: assert props["icon_name"] == "cloud_done" +def test_sync_status_acquiring_uses_muted() -> None: + """``acquiring`` (new file, still settling) uses the muted token.""" + + props = sync_status_icon.sync_status_props("acquiring") + assert props["color_var"] == "--color-muted" + assert props["status"] == "acquiring" + assert props["icon_name"] + + +def test_sync_status_syncing_uses_info() -> None: + """``syncing`` (settled, transferring) uses the info token.""" + + props = sync_status_icon.sync_status_props("syncing") + assert props["color_var"] == "--color-info" + assert props["status"] == "syncing" + assert props["icon_name"] + + +def test_sync_status_on_nas_uses_muted_cloud() -> None: + """``on_nas`` (tombstone, local copy cleared) uses a muted cloud glyph.""" + + props = sync_status_icon.sync_status_props("on_nas") + assert props["color_var"] == "--color-muted" + assert props["status"] == "on_nas" + assert props["icon_name"] == "cloud" + + def test_sync_status_retrying_with_counter() -> None: """Retry counter renders as ``(N/M)`` (Frontend §10.5.1).""" @@ -651,15 +678,19 @@ def test_tree_run_node_propagates_sync_status() -> None: assert nodes[0].children[0].children[0].sync_status == "cleaned" -def test_to_nicegui_nodes_cleaned_run_uses_cloud_icon() -> None: - """A ``cleaned`` run row carries the cloud-icon URL and its sync_status.""" +def test_to_nicegui_nodes_cleared_run_uses_cloud_icon() -> None: + """A ``cleared`` run row carries the cloud-icon URL and its sync_status. + + Operator-free per-file NAS sync design (2026-05-21): the run rollup + is a :class:`RunSyncState` value; the cloud icon keys off ``cleared``. + """ equipment = tree.EquipmentNode(equipment_id="CONFOCAL_01") project = tree.ProjectNode(short_id="PROJ-1", name="Cortex Q3") run = tree.RunNode( directory_name="Run_2026-05-07", run_kind="experimental", - sync_status="cleaned", + sync_status="cleared", ) payload = tree.to_nicegui_nodes( tree.build_nodes( @@ -669,7 +700,7 @@ def test_to_nicegui_nodes_cleaned_run_uses_cloud_icon() -> None: ) run_dict = payload[0]["children"][0]["children"][0] assert run_dict["sync_icon"] == tree.SYNC_ICON_CLOUD_URL - assert run_dict["sync_status"] == "cleaned" + assert run_dict["sync_status"] == "cleared" def test_to_nicegui_nodes_local_run_uses_local_icon() -> None: diff --git a/tests/unit/ui/test_dynamic_form.py b/tests/unit/ui/test_dynamic_form.py index 16ece77..8247af1 100644 --- a/tests/unit/ui/test_dynamic_form.py +++ b/tests/unit/ui/test_dynamic_form.py @@ -7,7 +7,7 @@ * ``render_question_field`` -- seeds the answers dict with each question's default (the only headlessly-assertable behaviour). * ``build_equipment_config`` -- the equipment-editor builder, across - both completeness signals and both transports. + both transports. """ from __future__ import annotations @@ -18,7 +18,6 @@ # Prime the api package before importing ui.pages (import-cycle workaround). import exlab_wizard.api.app # noqa: F401 from exlab_wizard.config.models import RcloneTransport, RsyncSshTransport -from exlab_wizard.constants import CompletenessSignal from exlab_wizard.ui.pages.settings import build_equipment_config from exlab_wizard.ui.pages.templates import ( TemplateQuestion, @@ -123,9 +122,6 @@ def _equipment_kwargs(**overrides: object) -> dict[str, object]: "label": "Confocal 1", "local_root": "/data/microscope1", "nas_root": "/nas/microscope1", - "completeness_signal": "sentinel_file", - "sentinel_filename": "done.flag", - "manifest_filename": "", "transport_type": "rclone", "rclone_remote": "lab-nas", "rclone_remote_path": "lab/microscope1", @@ -137,22 +133,16 @@ def _equipment_kwargs(**overrides: object) -> dict[str, object]: return base -def test_build_equipment_rclone_sentinel() -> None: +def test_build_equipment_rclone() -> None: entry = build_equipment_config(**_equipment_kwargs()) # type: ignore[arg-type] assert entry.id == "MICROSCOPE1" - assert entry.completeness_signal is CompletenessSignal.SENTINEL_FILE - assert entry.sentinel_filename == "done.flag" - assert entry.manifest_filename is None assert isinstance(entry.transport, RcloneTransport) assert entry.transport.rclone_remote == "lab-nas" -def test_build_equipment_rsync_manifest() -> None: +def test_build_equipment_rsync() -> None: entry = build_equipment_config( **_equipment_kwargs( # type: ignore[arg-type] - completeness_signal="manifest", - sentinel_filename="", - manifest_filename="manifest.json", transport_type="rsync_ssh", rclone_remote="", rclone_remote_path="", @@ -161,9 +151,6 @@ def test_build_equipment_rsync_manifest() -> None: rsync_remote_path="/remote/microscope1", ) ) - assert entry.completeness_signal is CompletenessSignal.MANIFEST - assert entry.manifest_filename == "manifest.json" - assert entry.sentinel_filename is None assert isinstance(entry.transport, RsyncSshTransport) assert entry.transport.ssh_target == "operator@host" assert entry.transport.remote_path == "/remote/microscope1" @@ -172,10 +159,3 @@ def test_build_equipment_rsync_manifest() -> None: def test_build_equipment_rejects_bad_id() -> None: with pytest.raises(ValidationError): build_equipment_config(**_equipment_kwargs(equipment_id="lower_case")) # type: ignore[arg-type] - - -def test_build_equipment_rejects_sentinel_without_filename() -> None: - with pytest.raises(ValidationError): - build_equipment_config( - **_equipment_kwargs(sentinel_filename="") # type: ignore[arg-type] - ) diff --git a/tests/unit/ui/test_file_list.py b/tests/unit/ui/test_file_list.py index d57ee19..1f48505 100644 --- a/tests/unit/ui/test_file_list.py +++ b/tests/unit/ui/test_file_list.py @@ -3,6 +3,9 @@ from __future__ import annotations from exlab_wizard.ui.components.file_list import ( + FILE_CONTEXT_COPY_PATH, + FILE_CONTEXT_KEEP_LOCAL, + FILE_CONTEXT_OPEN, FileListEntry, diff_file_lists, ) @@ -65,3 +68,81 @@ def test_diff_unchanged_is_empty() -> None: assert diff.added == () assert diff.removed == () assert diff.modified == () + + +# --------------------------------------------------------------------------- +# Keep-local / tombstone fields (operator-free per-file NAS sync design) +# --------------------------------------------------------------------------- + + +def test_file_list_entry_defaults_keep_local_and_tombstone_false() -> None: + """A bare entry defaults ``keep_local`` and ``tombstone`` to False.""" + entry = _entry("/r/scan.tif") + assert entry.keep_local is False + assert entry.tombstone is False + + +def test_diff_detects_keep_local_change_as_modification() -> None: + """Flipping ``keep_local`` is a modification (drives a re-render).""" + before = FileListEntry(name="a", path="/r/a", is_dir=False, keep_local=False) + after = FileListEntry(name="a", path="/r/a", is_dir=False, keep_local=True) + diff = diff_file_lists(previous=[before], current=[after]) + assert diff.modified == ("/r/a",) + + +def test_diff_detects_tombstone_change_as_modification() -> None: + """A file transitioning to a tombstone is a modification.""" + before = FileListEntry(name="a", path="/r/a", is_dir=False, tombstone=False) + after = FileListEntry(name="a", path="/r/a", is_dir=False, tombstone=True) + diff = diff_file_lists(previous=[before], current=[after]) + assert diff.modified == ("/r/a",) + + +def test_keep_local_action_constant_is_distinct() -> None: + """``FILE_CONTEXT_KEEP_LOCAL`` is a distinct discriminator value.""" + assert FILE_CONTEXT_KEEP_LOCAL not in {FILE_CONTEXT_OPEN, FILE_CONTEXT_COPY_PATH} + + +def test_render_file_list_keep_local_menu_and_tombstone() -> None: + """The renderer wires the keep-local menu item and tombstone row. + + Exercises the NiceGUI render path inside an app context so the + keep-local context-menu item and the tombstone (no Open-in-OS) row + actually build without raising. + """ + from nicegui import ui + + from exlab_wizard.ui.components.file_list import FileListState, render_file_list + + actions: list[tuple[str, str]] = [] + + @ui.page("/_test_file_list") # pragma: no cover -- render path + def _page() -> None: + state = FileListState( + path="/r", + entries=[ + FileListEntry( + name="scan.tif", + path="/r/scan.tif", + is_dir=False, + size_bytes=10, + sync_status="synced", + keep_local=True, + ), + FileListEntry( + name="old.tif", + path="/r/old.tif", + is_dir=False, + sync_status="on_nas", + tombstone=True, + ), + ], + ) + render_file_list( + state=state, + on_context_menu=lambda e, a: actions.append((e.path, a)), + ) + + # Building the page handler without raising is the assertion here; + # the e2e suite drives the live DOM interaction. + assert callable(_page) diff --git a/tests/unit/ui/test_mount.py b/tests/unit/ui/test_mount.py index f494618..f49a378 100644 --- a/tests/unit/ui/test_mount.py +++ b/tests/unit/ui/test_mount.py @@ -1038,7 +1038,6 @@ def test_build_metadata_payload_owned_equipment_reads_config() -> None: sync_mode="nas", local_root="/data/EQ1", nas_root="//nas/EQ1", - completeness_signal="sentinel_file", ) config = _config(equipment=(equipment,)) payload = mount._build_metadata_payload("EQ1", "equipment", _deps(config=config)) @@ -1047,7 +1046,6 @@ def test_build_metadata_payload_owned_equipment_reads_config() -> None: assert payload["sync_mode"] == "nas" assert payload["local_root"] == "/data/EQ1" assert payload["nas_root"] == "//nas/EQ1" - assert payload["completeness_signal"] == "sentinel_file" def test_build_metadata_payload_unknown_equipment_id_returns_empty() -> None: @@ -1232,7 +1230,7 @@ def test_file_context_action_open_in_os_dispatches_to_helper( monkeypatch.setattr(mount, "_open_in_os", lambda p: called.append(p) or True) ui = _UiSpy() entry = SimpleNamespace(path="/data/EQ1/scan.tif") - mount._file_context_action(entry, "open_in_os", ui) + mount._file_context_action(None, entry, "open_in_os", ui) assert called == ["/data/EQ1/scan.tif"] @@ -1243,7 +1241,7 @@ def test_file_context_action_open_in_os_failure_toasts_negative( monkeypatch.setattr(mount, "_open_in_os", lambda _p: False) ui = _UiSpy() entry = SimpleNamespace(path="/data/scan.tif") - mount._file_context_action(entry, "open_in_os", ui) + mount._file_context_action(None, entry, "open_in_os", ui) # No assertion needed -- the test only verifies the call doesn't raise. @@ -1251,7 +1249,7 @@ def test_file_context_action_copy_path_writes_clipboard() -> None: """``copy_path`` writes the entry path to the NiceGUI clipboard.""" ui = _UiSpy() entry = SimpleNamespace(path="/data/EQ1/scan.tif") - mount._file_context_action(entry, "copy_path", ui) + mount._file_context_action(None, entry, "copy_path", ui) assert ui.clipboard.writes == ["/data/EQ1/scan.tif"] @@ -1259,13 +1257,13 @@ def test_file_context_action_unknown_action_no_raise() -> None: """An unknown action verb is logged + toasted without raising.""" ui = _UiSpy() entry = SimpleNamespace(path="/data/x.bin") - mount._file_context_action(entry, "rename", ui) # no AssertionError + mount._file_context_action(None, entry, "rename", ui) # no AssertionError def test_file_context_action_empty_path_toasts_negative() -> None: """An entry with no path triggers the early-return toast.""" ui = _UiSpy() - mount._file_context_action(SimpleNamespace(path=""), "open_in_os", ui) + mount._file_context_action(None, SimpleNamespace(path=""), "open_in_os", ui) # --------------------------------------------------------------------------- @@ -1329,7 +1327,7 @@ def test_run_staging_action_view_log_invokes_log_dialog( ) -> None: """``view_log`` dispatches to the dialog opener helper.""" seen: list[Path] = [] - monkeypatch.setattr(mount, "_open_log_dialog", lambda path, _ui: seen.append(path)) + monkeypatch.setattr(mount, "_open_log_dialog", lambda _deps, path, _ui: seen.append(path)) deps = _deps(config=_config()) ui = _UiSpy() mount._run_staging_action(deps, "EQ1/proj/Run_x", "view_log", ui) @@ -1343,18 +1341,18 @@ def test_run_staging_action_unknown_action_no_raise() -> None: mount._run_staging_action(deps, "EQ1/proj/Run_x", "rename", ui) -async def test_run_staging_action_clear_verified_invokes_clear_run( +async def test_run_staging_action_clear_verified_invokes_clear( monkeypatch: pytest.MonkeyPatch, ) -> None: - """``clear_verified`` calls orchestrator.cleanup.clear_run via background.""" + """``clear_verified`` deletes the run directory via the local helper.""" captured: list[Path] = [] - async def _stub(run_path: Path, **_kw: Any) -> tuple[int, int]: + def _stub(run_path: Path) -> tuple[int, int]: captured.append(run_path) return 3, 1024 - monkeypatch.setattr("exlab_wizard.orchestrator.cleanup.clear_run", _stub) - deps = _deps(config=_config(), ingest_writer=None) + monkeypatch.setattr(mount, "clear_run_dir", _stub) + deps = _deps(config=_config()) ui = _UiSpy() mount._run_staging_action(deps, "EQ1/proj/Run_x", "clear_verified", ui) pending = [t for t in mount._BACKGROUND_TASKS if not t.done()] @@ -1368,25 +1366,33 @@ async def _stub(run_path: Path, **_kw: Any) -> tuple[int, int]: # --------------------------------------------------------------------------- -async def test_bulk_clear_verified_calls_orchestrator_helper( +async def test_bulk_clear_verified_clears_verified_rows( monkeypatch: pytest.MonkeyPatch, ) -> None: - """The bulk action awaits ``clear_all_verified`` and toasts the count.""" - calls: list[Any] = [] + """The bulk action clears every ``synced`` row and toasts the count.""" + cleared: list[Path] = [] - async def _stub(*, config: Any, ingest_writer: Any, host: Any = None) -> list[str]: - del ingest_writer, host - calls.append(config) - return ["/staging/EQ1/proj/Run_a", "/staging/EQ1/proj/Run_b"] + def _summary(path: str, state: str) -> SimpleNamespace: + return SimpleNamespace(path=path, current_state=state) - monkeypatch.setattr("exlab_wizard.orchestrator.cleanup.clear_all_verified", _stub) - deps = _deps(config=_config(), ingest_writer=None) + monkeypatch.setattr( + mount, + "list_staged_runs", + lambda **_kw: [ + _summary("/staging/EQ1/proj/Run_a", "synced"), + _summary("/staging/EQ1/proj/Run_b", "synced"), + _summary("/staging/EQ1/proj/Run_c", "syncing"), + ], + ) + monkeypatch.setattr(mount, "clear_run_dir", lambda p: cleared.append(p) or (1, 10)) + deps = _deps(config=_config()) ui = _UiSpy() mount._bulk_clear_verified(deps, ui) pending = [t for t in mount._BACKGROUND_TASKS if not t.done()] for task in pending: await task - assert len(calls) == 1 + # Only the two verified rows were cleared. + assert cleared == [Path("/staging/EQ1/proj/Run_a"), Path("/staging/EQ1/proj/Run_b")] def test_bulk_clear_verified_no_config_toasts_and_returns() -> None: @@ -1399,13 +1405,13 @@ def test_bulk_clear_verified_no_config_toasts_and_returns() -> None: async def test_bulk_clear_verified_logs_helper_exception( monkeypatch: pytest.MonkeyPatch, ) -> None: - """An exception from clear_all_verified is caught + toasted.""" + """An exception from the clear sweep is caught + toasted.""" - async def _raise(**_kw: Any) -> list[str]: + def _raise(**_kw: Any) -> list[Any]: raise RuntimeError("staging walker exploded") - monkeypatch.setattr("exlab_wizard.orchestrator.cleanup.clear_all_verified", _raise) - deps = _deps(config=_config(), ingest_writer=None) + monkeypatch.setattr(mount, "list_staged_runs", _raise) + deps = _deps(config=_config()) ui = _UiSpy() mount._bulk_clear_verified(deps, ui) pending = [t for t in mount._BACKGROUND_TASKS if not t.done()] @@ -1546,26 +1552,38 @@ def _raise(*_a: Any, **_kw: Any) -> None: # --------------------------------------------------------------------------- -# _open_log_dialog: missing ingest / malformed / dialog mounted +# _open_log_dialog: no queue job / queue job present # --------------------------------------------------------------------------- -def test_open_log_dialog_missing_ingest_toasts(tmp_path: Path) -> None: - """A run without an ingest.json file shows a negative toast.""" +async def test_open_log_dialog_no_queue_job_renders_dialog(tmp_path: Path) -> None: + """A run with no sync-queue job still renders a dialog without raising.""" ui = _UiSpy() - mount._open_log_dialog(tmp_path / "nope", ui) + deps = _deps(nas_sync=None) + mount._open_log_dialog(deps, tmp_path / "nope", ui) + for task in [t for t in mount._BACKGROUND_TASKS if not t.done()]: + await task -def test_open_log_dialog_malformed_ingest_toasts(tmp_path: Path) -> None: - """A corrupt ingest.json surfaces a parse-error toast without raising.""" - from exlab_wizard.constants import CACHE_DIR_NAME, INGEST_JSON_NAME +async def test_open_log_dialog_with_queue_job_renders_dialog(tmp_path: Path) -> None: + """A run with a sync-queue job renders its state without raising.""" + + class _Row: + state = SimpleNamespace(value="verified") + enqueued_at = "2026-05-01T10:00:00Z" + verified_at = "2026-05-01T10:35:00Z" + attempts = 1 + last_error = None + + class _Queue: + async def get_by_run_path(self, _path: Path) -> Any: + return _Row() - run_dir = tmp_path / "EQ1" / "PROJ" / "Run_x" - cache = run_dir / CACHE_DIR_NAME - cache.mkdir(parents=True) - (cache / INGEST_JSON_NAME).write_bytes(b"{not-valid") ui = _UiSpy() - mount._open_log_dialog(run_dir, ui) + deps = _deps(nas_sync=_Queue()) + mount._open_log_dialog(deps, tmp_path / "EQ1" / "Run_x", ui) + for task in [t for t in mount._BACKGROUND_TASKS if not t.done()]: + await task # --------------------------------------------------------------------------- @@ -1661,12 +1679,12 @@ async def enqueue(self, _path: Path) -> None: async def test_run_staging_action_clear_verified_exception_path_no_raise( monkeypatch: pytest.MonkeyPatch, ) -> None: - """When clear_run raises, the background task swallows + toasts.""" + """When the clear helper raises, the background task swallows + toasts.""" - async def _raise(*_a: Any, **_kw: Any) -> tuple[int, int]: + def _raise(*_a: Any, **_kw: Any) -> tuple[int, int]: raise RuntimeError("oh no") - monkeypatch.setattr("exlab_wizard.orchestrator.cleanup.clear_run", _raise) + monkeypatch.setattr(mount, "clear_run_dir", _raise) deps = _deps(config=_config()) ui = _UiSpy() mount._run_staging_action(deps, "EQ1/Run_x", "clear_verified", ui) @@ -1679,10 +1697,7 @@ async def test_run_staging_action_clear_verified_zero_files_branch( ) -> None: """The 0-file path reports ``already cleared`` rather than ``cleared N``.""" - async def _stub(*_a: Any, **_kw: Any) -> tuple[int, int]: - return 0, 0 - - monkeypatch.setattr("exlab_wizard.orchestrator.cleanup.clear_run", _stub) + monkeypatch.setattr(mount, "clear_run_dir", lambda *_a, **_kw: (0, 0)) deps = _deps(config=_config()) ui = _UiSpy() mount._run_staging_action(deps, "EQ1/Run_x", "clear_verified", ui) @@ -1702,4 +1717,4 @@ def write(self, _v: str) -> None: ui = _UiSpy() ui.clipboard = _BadClipboard() entry = SimpleNamespace(path="/data/scan.tif") - mount._file_context_action(entry, "copy_path", ui) + mount._file_context_action(None, entry, "copy_path", ui) diff --git a/tests/unit/ui/test_staging_page.py b/tests/unit/ui/test_staging_page.py index 18d67b9..9ef76ef 100644 --- a/tests/unit/ui/test_staging_page.py +++ b/tests/unit/ui/test_staging_page.py @@ -8,7 +8,7 @@ from __future__ import annotations -from exlab_wizard.constants import IngestState +from exlab_wizard.constants import RunSyncState from exlab_wizard.orchestrator.staging_query import StagedRunSummary from exlab_wizard.ui.pages.staging import ( STAGING_DOCK_HEIGHT_PX, @@ -99,11 +99,9 @@ def test_format_elapsed_handles_negative() -> None: def test_state_pill_props_returns_label_and_color_for_each_state() -> None: for state in ( - IngestState.STAGING, - IngestState.COMPLETE, - IngestState.SYNC_QUEUED, - IngestState.SYNC_VERIFIED, - IngestState.CLEARED, + RunSyncState.SYNCING, + RunSyncState.SYNCED, + RunSyncState.CLEARED, ): props = state_pill_props(state.value) assert props["label"] == state.value @@ -124,7 +122,7 @@ def test_state_pill_props_falls_back_for_unknown_state() -> None: def _make_row( *, - state: IngestState = IngestState.STAGING, + state: str = "syncing", path: str = "/staging/EQ1/PROJ-0001/Run_2026-04-17T14-32-00", files: int = 5, byte_total: int = 4096, @@ -132,7 +130,7 @@ def _make_row( ) -> StagedRunSummary: return StagedRunSummary( path=path, - current_state=state.value, + current_state=state, equipment_id="EQ1", project_name="PROJ-0001", run_kind="experimental", @@ -148,18 +146,16 @@ def test_row_props_emits_run_label_as_leaf() -> None: assert props["run_label"] == "Run_2026-04-17T14-32-00" -def test_row_props_marks_sync_verified_as_clearable() -> None: - props = row_props(_make_row(state=IngestState.SYNC_VERIFIED)) +def test_row_props_marks_synced_rollup_as_clearable() -> None: + """Only a fully-``synced`` run rollup is clearable.""" + props = row_props(_make_row(state=RunSyncState.SYNCED.value)) assert props["is_clearable"] is True -def test_row_props_does_not_mark_other_states_as_clearable() -> None: - for state in ( - IngestState.STAGING, - IngestState.COMPLETE, - IngestState.SYNC_QUEUED, - IngestState.CLEARED, - ): +def test_row_props_does_not_mark_other_rollups_as_clearable() -> None: + # ``cleared`` is excluded: that run's staging copy is already gone, so + # a "Clear" affordance would be a no-op. ``syncing`` is unproven. + for state in (RunSyncState.SYNCING.value, RunSyncState.CLEARED.value): props = row_props(_make_row(state=state)) assert props["is_clearable"] is False, state diff --git a/tests/unit/ui/test_wizard_equipment.py b/tests/unit/ui/test_wizard_equipment.py index fa4f144..e3ec840 100644 --- a/tests/unit/ui/test_wizard_equipment.py +++ b/tests/unit/ui/test_wizard_equipment.py @@ -6,12 +6,21 @@ from pydantic import ValidationError from exlab_wizard.ui.pages.wizard_equipment import ( + EQUIPMENT_STEP_TITLES, + EQUIPMENT_WIZARD_STEPS, EquipmentWizardState, assemble_equipment_config, can_advance, ) +def test_wizard_has_four_steps_without_signal_step() -> None: + """The completeness-signal step is removed by the quiescence redesign.""" + assert EQUIPMENT_WIZARD_STEPS == ("identity", "paths", "sync_mode", "review") + assert set(EQUIPMENT_STEP_TITLES) == set(EQUIPMENT_WIZARD_STEPS) + assert "signal" not in EQUIPMENT_WIZARD_STEPS + + def _state_filled_for(step: str) -> EquipmentWizardState: s = EquipmentWizardState(active_step=step) s.equipment_id = "FLOW_99" @@ -22,8 +31,6 @@ def _state_filled_for(step: str) -> EquipmentWizardState: s.transport_type = "rclone" s.rclone_remote = "lab-nas" s.rclone_remote_path = "lab/FLOW_99" - s.completeness_signal = "sentinel_file" - s.sentinel_filename = "done.flag" return s @@ -66,17 +73,6 @@ def test_can_advance_sync_mode_stage_requires_staging_fields() -> None: assert can_advance(s) is True -def test_can_advance_signal_requires_matching_filename() -> None: - s = _state_filled_for("signal") - assert can_advance(s) is True - s.completeness_signal = "manifest" - s.sentinel_filename = "" - s.manifest_filename = "" - assert can_advance(s) is False - s.manifest_filename = "manifest.json" - assert can_advance(s) is True - - def test_assemble_round_trips_to_valid_equipment_config_nas() -> None: s = _state_filled_for("review") eq = assemble_equipment_config(s) From e39e75d825f8a06b55439f5dd79accc40d6fb9b7 Mon Sep 17 00:00:00 2001 From: Alexander Nguyen Date: Mon, 25 May 2026 18:03:45 -0700 Subject: [PATCH 02/73] feat: TEST_-prefix mode for identifiable NAS test runs; GUI polish - Add EXLAB_WIZARD_TEST_MODE env (and unify it with the existing --test tray flag): when set, every loaded equipment.id gains a TEST_ prefix at config-load time. Run dirs and NAS paths inherit it, so test runs land under /TEST_/... -- trivially identifiable and bulk-deletable. Idempotent; re-validates the rewritten config. - TEST_MODE_ENV / TEST_MODE_PREFIX constants in constants/app.py, shared with paths.py. - Rename the test app's seeded equipment to TEST_EQ1 / TEST_RELAY_EQX to match the convention; cascade through 9 e2e flow files. - File list: add NAME / SIZE / MODIFIED / STATUS column headers in proper /. - Test app: keep-local context action now mutates the seeded feed and re-renders so the badge visibly toggles (production routes the same action through SyncStateWriter.set_keep_local). Co-Authored-By: Claude Opus 4.7 --- src/exlab_wizard/config/loader.py | 61 +++++++- src/exlab_wizard/constants/__init__.py | 4 +- src/exlab_wizard/constants/app.py | 18 +++ src/exlab_wizard/paths.py | 16 +- src/exlab_wizard/ui/components/file_list.py | 42 ++++- tests/e2e/_test_app.py | 95 +++++++---- tests/e2e/test_flow_01_onboarding.py | 2 +- tests/e2e/test_flow_05_browse_view.py | 2 +- tests/e2e/test_flow_16_add_equipment.py | 4 +- tests/e2e/test_flow_17_picker_step.py | 2 +- tests/e2e/test_flow_18_relay_receive.py | 2 +- tests/e2e/test_flow_19_travelling_badge.py | 2 +- tests/e2e/test_flow_20_file_explorer.py | 2 +- tests/e2e/test_flow_24_context_menus.py | 2 +- .../test_flow_25_production_main_wiring.py | 18 ++- tests/unit/config/test_loader.py | 147 ++++++++++++++++++ tests/unit/tray/test_main.py | 7 +- 17 files changed, 361 insertions(+), 65 deletions(-) diff --git a/src/exlab_wizard/config/loader.py b/src/exlab_wizard/config/loader.py index 85a8425..cd9516f 100644 --- a/src/exlab_wizard/config/loader.py +++ b/src/exlab_wizard/config/loader.py @@ -9,6 +9,7 @@ from __future__ import annotations import io +import os from pathlib import Path from typing import Any @@ -16,12 +17,64 @@ from ruamel.yaml import YAML from exlab_wizard.config.models import Config +from exlab_wizard.constants import TEST_MODE_ENV, TEST_MODE_PREFIX from exlab_wizard.errors import ConfigError from exlab_wizard.io import atomic_write_bytes from exlab_wizard.logging import get_logger _log = get_logger(__name__) +# Values that flip ``EXLAB_WIZARD_TEST_MODE`` on. Matched case-insensitively +# against the env var's stripped value; anything else (including unset, "", +# "0", "false", "no") leaves the loaded config untouched. +_TEST_MODE_TRUTHY: frozenset[str] = frozenset({"1", "true", "yes", "on"}) + + +def _test_mode_enabled() -> bool: + """Return True when the env var :data:`TEST_MODE_ENV` is truthy. + + Centralized here so callers don't have to know the truthy/falsy + spellings; see :data:`_TEST_MODE_TRUTHY` for the accepted strings. + """ + raw = os.environ.get(TEST_MODE_ENV, "") + return raw.strip().lower() in _TEST_MODE_TRUTHY + + +def apply_test_mode_prefix(config: Config) -> Config: + """Return ``config`` with each equipment id prefixed by :data:`TEST_MODE_PREFIX`. + + Idempotent: an id that already begins with the prefix is left + unchanged so repeated applications (or already-prefixed inputs) + never grow a ``TEST_TEST_…`` chain. The rewritten config is re-run + through :meth:`Config.model_validate` so unique-id and pattern + invariants are re-checked against the new ids -- if a rewrite ever + produces a duplicate or an over-length id, the loader raises the + same :class:`ConfigError` shape a hand-edited config would. + """ + if not config.equipment: + return config + + needs_rewrite = any(not entry.id.startswith(TEST_MODE_PREFIX) for entry in config.equipment) + if not needs_rewrite: + return config + + # Round-trip through ``model_dump`` so the rewritten dict is valid + # input for ``Config.model_validate`` (which re-runs every field-, + # model-, and cross-field validator, including the equipment-id + # regex, the max-length check, and the unique-id invariant). + dumped: dict[str, Any] = config.model_dump(mode="python") + for entry in dumped.get("equipment", []): + current_id = entry.get("id", "") + if isinstance(current_id, str) and not current_id.startswith(TEST_MODE_PREFIX): + entry["id"] = f"{TEST_MODE_PREFIX}{current_id}" + + try: + return Config.model_validate(dumped) + except ValidationError as exc: + raise ConfigError( + f"applying {TEST_MODE_PREFIX!r} prefix to equipment ids failed validation:\n{exc}" + ) from exc + def _yaml() -> YAML: """Build a configured ruamel.yaml instance. @@ -61,9 +114,15 @@ def load_config_from_text(text: str) -> Config: if not isinstance(data, dict): raise ConfigError("config.yaml top level must be a mapping") try: - return Config.model_validate(data) + config = Config.model_validate(data) except ValidationError as exc: raise ConfigError(f"config.yaml failed validation:\n{exc}") from exc + # Apply the ``EXLAB_WIZARD_TEST_MODE`` opt-in *after* validation so the + # base config is verified once on its own, and the prefix step's + # error path (duplicate ids, over-length) is reported separately. + if _test_mode_enabled(): + config = apply_test_mode_prefix(config) + return config def save_config(path: Path, config: Config, *, original_text: str | None = None) -> None: diff --git a/src/exlab_wizard/constants/__init__.py b/src/exlab_wizard/constants/__init__.py index c44da36..5b52ee9 100644 --- a/src/exlab_wizard/constants/__init__.py +++ b/src/exlab_wizard/constants/__init__.py @@ -10,7 +10,7 @@ from __future__ import annotations # ---- App-level identifiers ---- -from exlab_wizard.constants.app import APP_NAME +from exlab_wizard.constants.app import APP_NAME, TEST_MODE_ENV, TEST_MODE_PREFIX # ---- Enums (Backend §4.7, §4.9.1, §5.2, §6.2.4, §7, §8.1, §11.3, §13.3) ---- from exlab_wizard.constants.enums import ( @@ -224,6 +224,8 @@ "SYNC_STATE_JSON_VERSION", "TEMPLATE_QUESTION_ID_PATTERN", "TEMPLATE_QUESTION_ID_REGEX", + "TEST_MODE_ENV", + "TEST_MODE_PREFIX", "TEST_RUNS_DIR_NAME", "TEST_RUNS_JSON_NAME", "TEST_RUNS_JSON_VERSION", diff --git a/src/exlab_wizard/constants/app.py b/src/exlab_wizard/constants/app.py index 73b5472..9872071 100644 --- a/src/exlab_wizard/constants/app.py +++ b/src/exlab_wizard/constants/app.py @@ -13,3 +13,21 @@ # constants/app.py (this file) and keeping in sync with the cache # directory name in constants/filenames.py. APP_NAME: str = "exlab-wizard" + +# Runtime opt-in flag: when set to a truthy value the config loader +# prefixes every ``equipment[i].id`` with :data:`TEST_MODE_PREFIX` so +# the resulting on-disk + NAS run directories sort under a single +# ``TEST_/...`` namespace that operators can identify (and +# later delete) without sifting individual run leaves. The wizard's +# downstream code paths (path construction, run discovery, NAS sync +# targets) only ever see the prefixed ID, so no other module needs to +# know about test mode. Truthy values are ``"1"`` / ``"true"`` / +# ``"yes"`` / ``"on"`` (case-insensitive); any other value (including +# unset / empty) leaves IDs unchanged. +TEST_MODE_ENV: str = "EXLAB_WIZARD_TEST_MODE" + +# Equipment-ID prefix applied by :func:`exlab_wizard.config.loader.apply_test_mode_prefix` +# when :data:`TEST_MODE_ENV` is set to a truthy value. Must satisfy the +# equipment-ID regex (``^[A-Z][A-Z0-9_]*$``) so the prefixed value +# round-trips through the model's id validator. +TEST_MODE_PREFIX: str = "TEST_" diff --git a/src/exlab_wizard/paths.py b/src/exlab_wizard/paths.py index 477a683..a9b7ee4 100644 --- a/src/exlab_wizard/paths.py +++ b/src/exlab_wizard/paths.py @@ -31,6 +31,7 @@ RUN_DATE_STRFTIME, RUN_DIR_PREFIX, RUNS_DIR_NAME, + TEST_MODE_ENV, TEST_RUN_DIR_PREFIX, TEST_RUNS_DIR_NAME, WINDOWS_ILLEGAL_CHARS, @@ -78,11 +79,16 @@ # Setting ``EXLAB_WIZARD_TEST_MODE=1`` swaps APP_NAME for ``APP_NAME-test`` # in every OS-path helper below, redirecting config / state / cache / logs # into a parallel ``exlab-wizard-test`` sandbox without touching real user -# directories. The env var (rather than a CLI arg threaded through every -# layer) means the window subprocess spawned by WindowLauncher inherits the -# override automatically. See ``exlab-wizard-tray --test``. - -TEST_MODE_ENV = "EXLAB_WIZARD_TEST_MODE" +# directories. The same env var also drives ``apply_test_mode_prefix`` in +# ``config.loader`` so on-disk + NAS run directories sort under a +# ``TEST_/...`` namespace. The env var (rather than a CLI arg threaded +# through every layer) means the window subprocess spawned by +# WindowLauncher inherits the override automatically. See +# ``exlab-wizard-tray --test``. +# +# ``TEST_MODE_ENV`` is re-exported here for backward compat with callers +# that imported it from this module before it was centralized in +# ``constants/app.py``. def _app_name() -> str: diff --git a/src/exlab_wizard/ui/components/file_list.py b/src/exlab_wizard/ui/components/file_list.py index 9020dfa..e67e905 100644 --- a/src/exlab_wizard/ui/components/file_list.py +++ b/src/exlab_wizard/ui/components/file_list.py @@ -139,16 +139,44 @@ def render_file_list( .style("border-collapse: collapse; font-family: var(--font-mono);") .props('data-testid="file-list-table"') ): - for entry in state.entries: - _render_row( - entry, - is_new=entry.path in state.new_paths, - on_double_click=on_double_click, - on_context_menu=on_context_menu, - ) + with ui.element("thead"): + _render_header() + with ui.element("tbody"): + for entry in state.entries: + _render_row( + entry, + is_new=entry.path in state.new_paths, + on_double_click=on_double_click, + on_context_menu=on_context_menu, + ) return container +def _render_header() -> None: # pragma: no cover -- NiceGUI render, driven by e2e + """Render the file-list column header row (Name / Size / Modified / Status).""" + try: + from nicegui import ui + except Exception: + return + cell = ( + "font-size: var(--text-xs); text-transform: uppercase; letter-spacing: 0.06em; " + "color: var(--color-muted); font-weight: 600;" + ) + with ( + ui.element("tr") + .style("border-bottom: 1px solid var(--color-rule);") + .props('data-testid="file-list-header"') + ): + for title, align in ( + ("Name", "left"), + ("Size", "right"), + ("Modified", "left"), + ("Status", "left"), + ): + with ui.element("th").classes("p-2").style(f"text-align: {align}; {cell}"): + ui.label(title) + + def _render_row( entry: FileListEntry, *, diff --git a/tests/e2e/_test_app.py b/tests/e2e/_test_app.py index f28b6c8..4804ef5 100644 --- a/tests/e2e/_test_app.py +++ b/tests/e2e/_test_app.py @@ -115,7 +115,10 @@ def _classify_test_node(node_id: str) -> tuple[str, bool]: classify cleanly. """ if "/" not in node_id: - if node_id.startswith("RELAY_"): + # The test-app seed prefixes every equipment id with ``TEST_`` + # to mirror the production test-mode convention; the embedded + # ``RELAY_`` marker still identifies a relayed (received) root. + if "RELAY_" in node_id: return "received_equipment", True return "equipment", False if "Run_" in node_id or "TestRun_" in node_id: @@ -183,30 +186,38 @@ def _seeded_metadata_payload(node_id: str | None, node_kind: str | None) -> dict ] +def _feed_rows( + test_state: TestState, node_id: str +) -> list[tuple[str, int | None, str | None, bool, bool]]: + """Return a node's feed as normalized 5-tuples. + + ``(name, size, sync_status, keep_local, tombstone)``. A seeded + 3-tuple ``(name, size, sync)`` is widened with ``keep_local=False`` + / ``tombstone=False`` for backward compatibility. + """ + seeded = test_state.folder_feeds.get(node_id) + if seeded is None: + return list(_DEFAULT_FEED_ROWS) + return [ + (row[0], row[1], row[2], False, False) + if len(row) == 3 + else (row[0], row[1], row[2], row[3], row[4]) + for row in seeded + ] + + def _seeded_file_entries(test_state: TestState, node_id: str | None) -> list[Any]: """Build the centre-pane file rows the test flows assert on. Returns a default synthetic feed (covering every per-file display state, including a keep-local file and an "On NAS" tombstone) unless - the test seeded a specific path via ``test_state.folder_feeds``. A - seeded 3-tuple ``(name, size, sync)`` keeps backward compatibility; a - 5-tuple additionally carries ``keep_local`` / ``tombstone``. + the test seeded a specific path via ``test_state.folder_feeds``. """ if node_id is None: return [] from exlab_wizard.ui.components.file_list import FileListEntry - seeded = test_state.folder_feeds.get(node_id) - rows: list[tuple[str, int | None, str | None, bool, bool]] - if seeded is None: - rows = list(_DEFAULT_FEED_ROWS) - else: - rows = [ - (row[0], row[1], row[2], False, False) - if len(row) == 3 - else (row[0], row[1], row[2], row[3], row[4]) - for row in seeded - ] + rows = _feed_rows(test_state, node_id) return [ FileListEntry( name=name, @@ -301,12 +312,15 @@ def main_index( test_state.selected_node_kind = node_kind test_state.selected_node_is_received = is_received - # Hierarchy used by every /main test. Owned EQ1 carries the - # local + cleaned + test-run mix that flow 05b's sync-icon - # assertions depend on; the relay-flagged RELAY_EQX root - # surfaces the received-equipment row flow 18 / 24 target. + # Hierarchy used by every /main test. Owned TEST_EQ1 carries + # the local + cleaned + test-run mix that flow 05b's sync-icon + # assertions depend on; the relay-flagged TEST_RELAY_EQX root + # surfaces the received-equipment row flow 18 / 24 target. The + # ``TEST_`` prefix mirrors the production test-mode convention + # (see ``constants.TEST_MODE_PREFIX``) so the seeded display + # matches what a test-mode operator would see on the NAS. hierarchy: dict[Any, Any] = { - tree_component.EquipmentNode("EQ1", relay=False): { + tree_component.EquipmentNode("TEST_EQ1", relay=False): { tree_component.ProjectNode("LIMS-001", "Demo Project"): [ tree_component.RunNode("Run_2026-05-07", "experimental", "Demo run"), tree_component.RunNode( @@ -327,7 +341,7 @@ def main_index( # tree-node-run that would break flow_20 / flow_24's bare # ``.locator('[data-testid="tree-node-run"]')`` strict-mode # queries. - tree_component.EquipmentNode("RELAY_EQX", relay=True): { + tree_component.EquipmentNode("TEST_RELAY_EQX", relay=True): { tree_component.ProjectNode("PROJ-Relay", "Relayed Project"): [], }, } @@ -401,7 +415,22 @@ def _on_tree_context_action(node_id: str, action: str) -> None: ui.navigate.to(f"/settings?active=equipment&equipment_id={node_id}") def _on_file_context_action(entry: Any, action: str) -> None: + from exlab_wizard.ui.components.file_list import FILE_CONTEXT_KEEP_LOCAL + test_state.last_action = f"file.{action}:{entry.path}" + # Keep-local toggles the seeded feed and re-renders so the + # badge visibly flips; production routes the same action + # through SyncStateWriter.set_keep_local. + if action == FILE_CONTEXT_KEEP_LOCAL and selected_path is not None: + name = entry.path.rsplit("/", 1)[-1] + test_state.folder_feeds[selected_path] = [ + (n, s, sy, (not kl) if n == name else kl, ts) + for (n, s, sy, kl, ts) in _feed_rows(test_state, selected_path) + ] + qs = f"selected={selected_path}" + if right_pane: + qs += f"&right_pane={right_pane}" + ui.navigate.to(f"/main?{qs}") main_page.render_file_explorer_page( on_open_new_project=_on_open_new_project, @@ -434,7 +463,7 @@ def project_wizard_index() -> None: s = wizard_project_page.ProjectWizardState( selected_lims_short_id="LIMS-001", selected_template="default", - selected_equipment="EQ1", + selected_equipment="TEST_EQ1", template_variables={}, readme_fields={"label": "demo", "operator": "asmith", "objective": "demo run"}, ) @@ -442,7 +471,7 @@ def project_wizard_index() -> None: def _submit(state: wizard_project_page.ProjectWizardState) -> None: test_state.last_action = "wizard.project.submit" # Render a confirm-card stand-in so tests see the success path - ui.label("Project created at /tmp/data/EQ1/LIMS-001").props( + ui.label("Project created at /tmp/data/TEST_EQ1/LIMS-001").props( 'data-testid="wizard-project-success"' ) @@ -458,7 +487,7 @@ def run_wizard_index() -> None: s = wizard_run_page.RunWizardState( run_kind="experimental", selected_project_name="Demo Project", - selected_equipment="EQ1", + selected_equipment="TEST_EQ1", selected_template="default", template_variables={}, readme_fields={"label": "demo", "operator": "asmith", "objective": "demo run"}, @@ -480,7 +509,7 @@ def test_run_wizard_index() -> None: s = wizard_run_page.RunWizardState( run_kind="test", selected_project_name="Demo Project", - selected_equipment="EQ1", + selected_equipment="TEST_EQ1", selected_template="default", template_variables={}, readme_fields={"label": "demo", "operator": "asmith", "objective": "demo run"}, @@ -611,10 +640,10 @@ def problems_index(seed: str = "", reset: int = 0) -> None: finding_id="F-1", severity=Tier.HARD, rule_class="Placeholder", - path="/data/EQ1/LIMS-001/Run_2026-05-07", + path="/data/TEST_EQ1/LIMS-001/Run_2026-05-07", matched_token="", run_label="Run_2026-05-07", - equipment="EQ1", + equipment="TEST_EQ1", detected_at="2026-05-07T10:00:00Z", state="Active", ), @@ -625,10 +654,10 @@ def problems_index(seed: str = "", reset: int = 0) -> None: finding_id="F-2", severity=Tier.HARD, rule_class="Missing field", - path="/data/EQ1/LIMS-001/Run_2026-05-07/.exlab-wizard/creation.json", + path="/data/TEST_EQ1/LIMS-001/Run_2026-05-07/.exlab-wizard/creation.json", matched_token="schema_version=2.0", run_label="Run_2026-05-07", - equipment="EQ1", + equipment="TEST_EQ1", detected_at="2026-05-07T10:00:00Z", state="Active", ), @@ -639,10 +668,10 @@ def problems_index(seed: str = "", reset: int = 0) -> None: finding_id="F-3", severity=Tier.HARD, rule_class="Orphan", - path="/data/EQ1/LIMS-001/Run_2026-05-07-orphan", + path="/data/TEST_EQ1/LIMS-001/Run_2026-05-07-orphan", matched_token="missing creation.json", run_label="Run_2026-05-07-orphan", - equipment="EQ1", + equipment="TEST_EQ1", detected_at="2026-05-07T10:00:00Z", state="Active", ), @@ -713,8 +742,8 @@ def staging_index(state: str = "none") -> None: rows = [ StagedRunSummary( - path="/staging/EQ1/LIMS-001/Run_2026-05-07", - equipment_id="EQ1", + path="/staging/TEST_EQ1/LIMS-001/Run_2026-05-07", + equipment_id="TEST_EQ1", project_name="LIMS-001", run_kind="experimental", current_state=state, diff --git a/tests/e2e/test_flow_01_onboarding.py b/tests/e2e/test_flow_01_onboarding.py index 5ff0544..04988f8 100644 --- a/tests/e2e/test_flow_01_onboarding.py +++ b/tests/e2e/test_flow_01_onboarding.py @@ -61,7 +61,7 @@ def test_flow_01_onboarding(page, server_url) -> None: page.locator('[data-testid="settings-paths-local-root"]').fill("/tmp/data") _goto(page, f"{server_url}/settings?incomplete=paths,equipment&active=equipment") - page.locator('[data-testid="settings-equipment-id"]').fill("EQ1") + page.locator('[data-testid="settings-equipment-id"]').fill("TEST_EQ1") page.locator('[data-testid="settings-equipment-add"]').click() # 5. Save and confirm setup banner clears. diff --git a/tests/e2e/test_flow_05_browse_view.py b/tests/e2e/test_flow_05_browse_view.py index be0b267..425121a 100644 --- a/tests/e2e/test_flow_05_browse_view.py +++ b/tests/e2e/test_flow_05_browse_view.py @@ -33,4 +33,4 @@ def test_flow_05_browse_view(page, server_url) -> None: main.footer_clear_verified.wait_for(state="visible") # Tree contains the seeded equipment label. - assert page.locator('[data-testid="main-tree"]').inner_text().find("EQ1") >= 0 + assert page.locator('[data-testid="main-tree"]').inner_text().find("TEST_EQ1") >= 0 diff --git a/tests/e2e/test_flow_16_add_equipment.py b/tests/e2e/test_flow_16_add_equipment.py index 905f6b9..78c5c70 100644 --- a/tests/e2e/test_flow_16_add_equipment.py +++ b/tests/e2e/test_flow_16_add_equipment.py @@ -82,7 +82,7 @@ def test_flow_16_next_enables_on_valid_input_and_state_survives_back(page, serve # Bug A: Next is gated shut until the step validates. expect(wiz.next_button).to_be_disabled() - wiz.equipment_id.fill("EQ1") + wiz.equipment_id.fill("TEST_EQ1") wiz.label.fill("Lab Device") expect(wiz.next_button).to_be_enabled() @@ -93,4 +93,4 @@ def test_flow_16_next_enables_on_valid_input_and_state_survives_back(page, serve # Bug B: stepping back keeps what the operator already typed. wiz.back.click() wiz.step_identity.wait_for(state="visible", timeout=10_000) - expect(wiz.equipment_id).to_have_value("EQ1") + expect(wiz.equipment_id).to_have_value("TEST_EQ1") diff --git a/tests/e2e/test_flow_17_picker_step.py b/tests/e2e/test_flow_17_picker_step.py index a958955..c1fd214 100644 --- a/tests/e2e/test_flow_17_picker_step.py +++ b/tests/e2e/test_flow_17_picker_step.py @@ -44,7 +44,7 @@ def test_flow_17_creation_buttons_disabled_on_received_node(page, server_url) -> # The click-then-navigate path races against Playwright's # networkidle wait under CI, where the URL doesn't change and the # state-mutation render may not finish before the assertion. - _goto(page, f"{server_url}/main?view=explorer&selected=RELAY_EQX") + _goto(page, f"{server_url}/main?view=explorer&selected=TEST_RELAY_EQX") for testid in ("toolbar-new-project", "toolbar-new-run", "toolbar-new-test-run"): btn = page.locator(f'[data-testid="{testid}"]') btn.wait_for(state="visible", timeout=10_000) diff --git a/tests/e2e/test_flow_18_relay_receive.py b/tests/e2e/test_flow_18_relay_receive.py index fdaf351..45abfb6 100644 --- a/tests/e2e/test_flow_18_relay_receive.py +++ b/tests/e2e/test_flow_18_relay_receive.py @@ -25,7 +25,7 @@ def test_flow_18_received_equipment_node_appears(page, server_url) -> None: _goto(page, f"{server_url}/main?view=explorer") node = page.locator('[data-testid="tree-node-received_equipment"]') node.wait_for(state="visible", timeout=10_000) - assert "RELAY_EQX" in node.inner_text() + assert "TEST_RELAY_EQX" in node.inner_text() def test_flow_18_received_metadata_shows_relay_badge(page, server_url) -> None: diff --git a/tests/e2e/test_flow_19_travelling_badge.py b/tests/e2e/test_flow_19_travelling_badge.py index e62657d..bf376e7 100644 --- a/tests/e2e/test_flow_19_travelling_badge.py +++ b/tests/e2e/test_flow_19_travelling_badge.py @@ -90,7 +90,7 @@ def test_flow_19_travelling_badge_aggregates_red(page, server_url) -> None: # test app reads. Falling back: the test app supports a ?seed= # query that the file-explorer view parses for findings. page.goto( - f"{server_url}/main?view=explorer&seed_finding=EQ1/PROJ-0001/Runs/Run_2026-05-14T09-22:hard", + f"{server_url}/main?view=explorer&seed_finding=TEST_EQ1/PROJ-0001/Runs/Run_2026-05-14T09-22:hard", wait_until="domcontentloaded", ) page.wait_for_load_state("networkidle") diff --git a/tests/e2e/test_flow_20_file_explorer.py b/tests/e2e/test_flow_20_file_explorer.py index e3ad790..627ec18 100644 --- a/tests/e2e/test_flow_20_file_explorer.py +++ b/tests/e2e/test_flow_20_file_explorer.py @@ -58,5 +58,5 @@ def test_flow_20_breadcrumb_is_present_when_node_selected(page, server_url) -> N page.wait_for_url(lambda url: "selected=" in url, timeout=10_000) page.locator('[data-testid="breadcrumb"]').wait_for(state="visible", timeout=5_000) segments = page.locator('[data-testid="breadcrumb-segment"]') - # selecting the run leaf gives EQ1 / Demo Project / Run_... + # selecting the run leaf gives TEST_EQ1 / Demo Project / Run_... assert segments.count() >= 1 diff --git a/tests/e2e/test_flow_24_context_menus.py b/tests/e2e/test_flow_24_context_menus.py index 87e362a..5209c13 100644 --- a/tests/e2e/test_flow_24_context_menus.py +++ b/tests/e2e/test_flow_24_context_menus.py @@ -57,7 +57,7 @@ def test_flow_24_edit_equipment_deep_links_into_settings(page, server_url) -> No page.locator('[data-testid="tree-context-edit-equipment"]').click() page.wait_for_url(lambda url: "/settings" in url, timeout=10_000) assert "active=equipment" in page.url - assert "equipment_id=EQ1" in page.url + assert "equipment_id=TEST_EQ1" in page.url def test_flow_24_remove_equipment_deep_links_into_settings(page, server_url) -> None: diff --git a/tests/e2e/test_flow_25_production_main_wiring.py b/tests/e2e/test_flow_25_production_main_wiring.py index 74a09aa..dc05cef 100644 --- a/tests/e2e/test_flow_25_production_main_wiring.py +++ b/tests/e2e/test_flow_25_production_main_wiring.py @@ -93,10 +93,10 @@ def test_flow_25_select_node_threads_selected_into_url(page, server_url) -> None def test_flow_25_selected_query_renders_centre_and_right_panes(page, server_url) -> None: - """Loading /main?selected=EQ1 directly renders the centre + right panes.""" - _goto(page, f"{server_url}/main?selected=EQ1") + """Loading /main?selected=TEST_EQ1 directly renders the centre + right panes.""" + _goto(page, f"{server_url}/main?selected=TEST_EQ1") # Centre pane: seeded folder feed shows the default two rows when no - # specific path is seeded for EQ1. + # specific path is seeded for TEST_EQ1. page.locator('[data-testid="file-list-row"]').first.wait_for(state="visible", timeout=5_000) # Right pane (metadata tab) renders with the equipment payload. page.locator('[data-testid="metadata-pane"]').wait_for(state="visible", timeout=5_000) @@ -109,7 +109,7 @@ def test_flow_25_breadcrumb_navigation_re_navigates_main(page, server_url) -> No clickable; wired to the same callback as on_select_node so the URL flow is identical. """ - _goto(page, f"{server_url}/main?selected=EQ1/Demo Project") + _goto(page, f"{server_url}/main?selected=TEST_EQ1/Demo Project") page.locator('[data-testid="breadcrumb"]').wait_for(state="visible", timeout=5_000) segments = page.locator('[data-testid="breadcrumb-segment"]') assert segments.count() >= 1 @@ -120,7 +120,7 @@ def test_flow_25_breadcrumb_navigation_re_navigates_main(page, server_url) -> No def test_flow_25_toggle_right_pane_toggles_query_param(page, server_url) -> None: """Clicking the right-pane toggle flips ?right_pane=collapsed in the URL.""" - _goto(page, f"{server_url}/main?selected=EQ1") + _goto(page, f"{server_url}/main?selected=TEST_EQ1") page.locator('[data-testid="toggle-right-pane"]').wait_for(state="visible", timeout=5_000) page.locator('[data-testid="toggle-right-pane"]').click() page.wait_for_url(lambda url: "right_pane=collapsed" in url, timeout=10_000) @@ -139,17 +139,19 @@ def test_flow_25_tree_context_action_deep_links_into_settings(page, server_url) eq.wait_for(state="visible", timeout=10_000) eq.click(button="right") page.locator('[data-testid="tree-context-edit-equipment"]').click() - page.wait_for_url(lambda url: "/settings" in url and "equipment_id=EQ1" in url, timeout=10_000) + page.wait_for_url( + lambda url: "/settings" in url and "equipment_id=TEST_EQ1" in url, timeout=10_000 + ) def test_flow_25_received_equipment_disables_creation_buttons(page, server_url) -> None: - """Selecting a RELAY_* node disables the New Project/Run/Test-Run buttons. + """Selecting a TEST_RELAY_* node disables the New Project/Run/Test-Run buttons. Verifies MainPageState.selected_node_is_received flows from the URL -> _classify_node -> render_file_explorer_page's disable logic (Redesign §3.3 / decision 1). """ - _goto(page, f"{server_url}/main?selected=RELAY_EQX") + _goto(page, f"{server_url}/main?selected=TEST_RELAY_EQX") # Quasar's q-btn ``disable`` prop renders as ``aria-disabled="true"`` # on the wrapper element; the inner