diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb511c..4205cb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## 0.4.0 (2026-10-22) + +cellmap-analyze's three asks, so it can call worldarray directly with no wrapper, and two +of cellmap-flow's. No placement change and no unit table change. + +- `vol.read_onto(grid, begin=None, end=None, ...)` reads voxels `[begin, end)` of a + `Grid`'s spatial axes from the array, exactly that shape (Behaviour 20). It is the + `read` of the grid's index box at the grid's voxel size anchored at its corner, so at + the array's own voxel size nothing is resampled, whatever the sub-voxel offset, and + otherwise the array is resampled onto the grid. The grid's units are converted. +- A pickled handle, and `reopen`, use the path resolved against the working directory at + open, so a worker that changes directory before unpickling opens the same store; it + used to raise `FileNotFoundError` for a relative path. +- `write_index`, `write` and `write_chunk` take a scalar, which fills the box. It is + converted to the dtype only when its value survives (`0` into `uint8`; `300`, `-1`, `0.5` + and NaN raise), and the fill value over whole units leaves them absent. +- `vol.source_translation`: the stored level's translation in `units`, beside + `source_voxel_size`, under resample and relabel too (cellmap-flow's `stored_placement`). +- `create(exists="adopt")` also compares the keys of `attrs` with the existing array's + attributes and raises naming each that differs or is missing; keys not given are not + compared. An adopt given different `attrs` used to keep the old ones silently, as funlib's + `prepare_ds` kept an old offset (cellmap-flow's `_refuse_another_region`). + ## 0.3.1 (2026-10-22) - `source_voxel_size` reports the stored level's voxel size under `on_mismatch="relabel"`, diff --git a/README.md b/README.md index 6b02152..556360c 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ six = wa.open(path, voxel_size=(6, 6, 6), on_mismatch="resample") # one volume read onto another volume's grid, for masks, contact sites, statistics onto = raw.read(box, voxel_size=labels.voxel_size, anchor=labels.corner) +onto = raw.read_onto(grid, begin, end) # voxels [begin, end) of a Grid, exactly that shape # index space, native units: what an engine or a viewer wants vol = wa.open(path, units=None) @@ -100,8 +101,8 @@ s1 = wa.open_pyramid("out.zarr", mode="r+").add_level("s1", factors=2) # place - **Plans outputs and blocks** (0.3): a store-less `Grid` with the same geometry as an array, so an output is planned, cut into blocks and only then created; `blocks` tiling an array or a box into write boxes with context-grown read boxes, in C order, as plain - picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule. - Nothing is scheduled or run. + picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule; every + input read onto that grid by its voxel indices (0.4). Nothing is scheduled or run. ## The placement rule @@ -168,12 +169,12 @@ wa.units.abbreviation("nanometer") ; wa.store.join(path, "s0") ; wa.store.parent ``` `WorldArray`: `ts`, `meta`, `grid`, `shape`, `dtype`, `axes`, `voxel_size`, `translation`, -`corner`, `box`, `levels`, `level_name`, `level_path`, `source_voxel_size`, `resampled`, +`corner`, `box`, `levels`, `level_name`, `level_path`, `source_voxel_size`, `source_translation`, `resampled`, `writable`, `write_shape`; `read(box, voxel_size=, padding=, method=, mask=, anchor=, transform=, post_transform=, select=, strict=)`, -`read_index(begin, end, padding=, mask=, select=, transform=)`, `index_box(box)`, +`read_index(begin, end, padding=, mask=, select=, transform=)`, `read_onto(grid, begin=, end=, ...)`, `index_box(box)`, `chunks_covering(begin, end, write=)`, `list_chunks()`, `chunk_key(index)`, `parse_chunk_key(key)`, -`write_index(begin, end, data, partial=)`, `write(box, data, partial=)`, `reopen(context=)`. +`write_index(begin, end, data, partial=)` and `write(box, data, partial=)` (`data` an array or a scalar), `reopen(context=)`. A box is a `(begin, end)` pair of floats in your units, half-open per axis, over the spatial axes or every axis; any object with `.begin`/`.end` or `.offset`/`.shape` also diff --git a/docs/spec.md b/docs/spec.md index 5744526..c656faa 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -149,8 +149,9 @@ vol.meta # ArrayMeta in the store's own units vol.grid # ArrayMeta in the requested units, every axis vol.shape, vol.dtype, vol.axes, vol.spatial, vol.level_name, vol.levels, vol.remote # shape is the stored shape, or with a resample target the target voxels covering the data -vol.source_voxel_size, vol.resampled # the stored level's voxel size, also under resample and relabel; whether reads - # resample to a target +vol.source_voxel_size, vol.source_translation, vol.resampled + # the stored level's voxel size and translation (0.4), also under resample and relabel; whether + # reads resample to a target vol.list_chunks() # the chunk indices that exist in storage, by listing keys (zarr v2/v3, N5, unsharded precomputed) vol.path # what the caller passed (a group, say); vol.level_path is the array actually opened vol.voxel_size # tuple[float] in `units`, spatial axes, the array's order (zyx for a volume) @@ -285,7 +286,8 @@ vol = wa.create( attrs=None, # extra array attributes, merged into the array's own group_attrs=None, # extra group attributes, merged; no one else's key is dropped or rewritten (Behaviour 12) exists="error", # "error": anything already at `path` raises | "adopt": an identical array (shape, dtype, chunks, - # shards, placement within rtol 1e-6) is opened for writing; a different one raises naming each + # shards, placement within rtol 1e-6, and from 0.4 every key of `attrs` holding the value given) + # is opened for writing; a different one raises naming each # difference. There is no third value. An adopt writes no group document when the entry # already says the same (Behaviour 12) store_fill=False, # True stores write units equal to fill_value instead of leaving them absent @@ -337,6 +339,20 @@ wa.store.join(path, *parts) ; wa.store.parent(path) ; wa.store.basename(path) wa.units.abbreviation(unit) -> str # the OME short form, "nm" for any spelling of nanometre the table knows ``` +### 0.4: reading onto a grid + +```python +vol.read_onto(grid, begin=None, end=None, padding=None, method=None, mask=False, retries=None, transform=None, + post_transform=None, select=None) + # voxels [begin, end) of a Grid's spatial axes (all of it when None), read from this array: + # exactly that shape, by the centre rule at the array's voxel size, else resampled onto the + # grid anchored at its corner (Behaviour 20) +vol.write_index(begin, end, 0) ; vol.write(box, 0) ; vol.write_chunk(index, 0) + # a scalar fills the box, converted to the dtype only when its value survives (Behaviour 13) +vol.source_translation # the stored level's translation in `units`, beside source_voxel_size, under resample and relabel +wa.create(..., attrs={...}, exists="adopt") # an adopt also compares the keys of `attrs` (Behaviour 11) +``` + ## 3. Conventions These are the rules the package exists to hold in one place. They match chunkmirage's @@ -682,9 +698,11 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3. compression; cellmap-flow's blosc stays blosc by passing it. There is no overwrite: with `exists="error"` any document already at `path` raises `FileExistsError`; with `exists="adopt"` an array whose shape, dtype, chunks, shards and placement match - (placement within `rtol 1e-6`) is opened for writing and its dataset entry ensured, and - one that differs raises `ValueError` listing every differing field with the value found - and the value wanted (cellmap-flow's `_refuse_another_region`, registration-station's + (placement within `rtol 1e-6`), and whose attributes hold every key of `attrs` with the + value given as JSON stores it (keys not given are someone else's and not compared), is + opened for writing and its dataset entry ensured, and one that differs raises + `ValueError` listing every differing field and key with the value found and the value + wanted (cellmap-flow's `_refuse_another_region`, registration-station's missing check, which let a shorter last band's chunk grid win a creation race). A creation that loses a race, the store reporting the array present after the probe said absent, is re-read with backoff, 5 attempts at `0.5 s * attempt`, and then treated as an @@ -697,7 +715,9 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3. being the one it replaces. `create` returns a writable `WorldArray`; `wa.open(path, mode="r+")` opens an existing one with every rule of Conventions 3 to 5 unchanged, and a handle pickles by its arguments and its context's spec, so each worker - process opens its own cheaply, on the shared context when the handle had one, else on + process opens its own cheaply (a local path goes resolved against the working directory + at open, so a worker that changes directory before unpickling, as cellmap-analyze's dask + workers do, opens the same store; `reopen` uses it too), on the shared context when the handle had one, else on a context rebuilt once per process from the same spec, so a capped context stays capped (mesh-n-bone's per-process cache and thread caps). @@ -722,7 +742,12 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3. 13. **Aligned writes need no coordination.** `write_index(begin, end, data)` requires `data.shape == end - begin`, `data.dtype == vol.dtype` (no silent cast; `ValueError`), - and `[begin, end)` to be a union of whole write units: every bound a multiple of + or `data` a scalar (a number or a 0-d array) that fills the box: it is converted to the + dtype only when its value survives, so an integer or bool array takes whole numbers in + its range and a float array any number it does not overflow, rounded to its precision + (`0` into `uint8` is written; `300`, `-1`, `0.5` and NaN raise), and a scalar equal to + `fill_value` over whole units leaves them absent as an all-fill array does (cellmap-analyze + writes a constant for every empty or masked-out block). The box must be a union of whole write units: every bound a multiple of `write_shape` on its axis or the array's end on that axis, so the clipped last unit is allowed; otherwise `ValueError` names the axis, the bound and the unit size. A unit written whole is one put, so writers in different processes writing different units @@ -809,6 +834,22 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3. returns the OME short form for any spelling the table knows and raises for one it does not. +20. **Reading onto a grid.** `vol.read_onto(grid, begin, end)` returns voxels `[begin, end)` + of a `Grid`'s spatial axes, all of it when `begin` and `end` are None, read from this + array, with exactly `end - begin` voxels on those axes; the array's other axes are read + as `read` reads them (whole, or pinned by `select`), the open-time `padding`, `method`, + `retries` and `transform` apply, and `mask` marks the grid voxels that held data. It is + `read(Box.from_index(begin, end, grid voxel size, grid translation), voxel_size=grid + voxel size, anchor=grid corner)`, so Behaviours 2 and 4 decide every voxel: at the + array's own voxel size (within level selection's rtol 1e-6) nothing is resampled, + whatever the offset between the grids, and each grid voxel takes the stored voxel whose + centre lies in it, never an interpolation, with `method="linear"` too; at any other + voxel size the array is resampled onto the grid, anchored at its corner. Adjoining + `[begin, end)` reads therefore agree with one read across both. The grid's spatial axes + must be named as the array's are (`ValueError` otherwise), and its units are converted + to the array's `units`. This is cellmap-analyze's per-operation grid: blocks come from + one `Grid`, every input is read onto it, and outputs are created on it. + ## 5. Tests (behavioural; synthetic stores in `tmp_path`; one optional network test) Conformance, one tiny store each, checking `voxel_size`, `translation`, `axes`, @@ -898,8 +939,10 @@ factor; missing ancestor documents down from the `.zarr` root and none above it, ancestor's format followed and a contradiction refused, an array in the way refused; an adopt writing no document and a changed placement writing one; a lost entry written again and the error after the attempts; `ensure_group_entry` on a foreign array, idempotent; a -pickled handle keeping a capped context's spec and sharing the shared one; `codecs_of` by -path; an all-fill write removing its own key. +pickled handle keeping a capped context's spec and sharing the shared one, and a relative +path reopening after a `chdir`; `codecs_of` by path; an all-fill write removing its own key; +a scalar filling a box through `write_index`, `write` and `write_chunk`, refused when it +does not fit the dtype. Grids (0.3): `blocks` partitioning with and without a box, `context` as an int, per axis and as before/after pairs, clipping on and off, `len`, `blocks[id]` and `index_of`; @@ -908,6 +951,14 @@ and as before/after pairs, clipping on and off, `len`, `blocks[id]` and `index_o reproducing mesh-n-bone's anisotropic table (8, 8, 30 nm: z stays at 1 until y and x reach 32) and all 2s for isotropic data; `abbreviation` for every spelling in the table. +Reading onto a grid (0.4): `read_onto` at the array's voxel size returning stored values +under a sub-voxel offset with `method="linear"`, and the next voxel past a half-voxel one; +onto a finer grid equal to the `read` it is defined as, in micrometres too, adjoining +reads agreeing, the mask marking grid voxels before the data; its own grid at voxel sizes +3.54, 4 and 5.3 returning the stored array exactly; a grid with other axis names refused. +`source_translation` under relabel and resample and in micrometres; an adopt keeping keys +it was not given and refusing a different or missing one, naming both values. + Target: about 600 lines of test cases for 0.1 and about 300 more for 0.2 and 0.3, the store builders in `conftest.py` aside; no wording pins; no `/nrs`; no cluster. diff --git a/pyproject.toml b/pyproject.toml index 6b81ce4..dbf4ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "worldarray" -version = "0.3.1" +version = "0.4.0" description = "Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only." readme = "README.md" license = { file = "LICENSE" } diff --git a/src/worldarray/__init__.py b/src/worldarray/__init__.py index 350f76b..37e8d67 100644 --- a/src/worldarray/__init__.py +++ b/src/worldarray/__init__.py @@ -28,7 +28,7 @@ parse_chunk_key, ) -__version__ = "0.3.1" +__version__ = "0.4.0" __all__ = [ "ArrayMeta", diff --git a/src/worldarray/array.py b/src/worldarray/array.py index 5988f54..cff63c7 100644 --- a/src/worldarray/array.py +++ b/src/worldarray/array.py @@ -36,7 +36,15 @@ read_meta_light, ) from .retry import Retry, read_with_retries -from .store import Node, detect, is_remote, join, open_array, precomputed_scale_index +from .store import ( + Node, + detect, + is_remote, + join, + normalize_path, + open_array, + precomputed_scale_index, +) logger = logging.getLogger(__name__) @@ -90,6 +98,9 @@ def __init__( if mode not in ("r", "r+"): raise ValueError("mode is 'r' (read) or 'r+' (read and write)") self.path = path + # where later opens find it (reopen, a pickle, the pyramid): a local path resolved + # against the working directory now, so a worker that changes directory opens the same store + self._where = normalize_path(path) self._mode = mode self._store_fill = bool(store_fill) self.padding = padding @@ -139,8 +150,9 @@ def __init__( self._meta = self._rescaled_to_base(base_voxel_size, base_units) space, time = _unit_targets(units) self._grid = self._meta.in_units(space=space, time=time, strict=strict_units) - # the stored level's spatial voxel size, before any relabel renames the grid + # the stored level's spatial placement, before any relabel renames the grid self._stored_voxel_size = tuple(self._grid.voxel_size[i] for i in self._grid.spatial) + self._stored_translation = tuple(self._grid.translation[i] for i in self._grid.spatial) self._target: tuple[float, ...] | None = None if voxel_size is not None: @@ -305,6 +317,13 @@ def source_voxel_size(self) -> tuple[float, ...]: is, whether reads resample it or ``on_mismatch="relabel"`` renames its grid.""" return self._stored_voxel_size + @property + def source_translation(self) -> tuple[float, ...]: + """The stored level's translation (the centre of its voxel 0) on the spatial axes, in + ``units``: where the level is, whether reads resample it or ``on_mismatch="relabel"`` + renames its grid.""" + return self._stored_translation + @property def resampled(self) -> bool: """Whether reads resample to a target voxel size (``on_mismatch="resample"``).""" @@ -373,7 +392,7 @@ def levels(self) -> list[tuple[str, tuple[float, ...]]]: def levels_pyramid(self) -> Pyramid: if self._pyramid is None: self._pyramid = Pyramid( - self.path, context=self.context, units=self.units, select=self._select + self._where, context=self.context, units=self.units, select=self._select ) return self._pyramid @@ -422,14 +441,15 @@ def reopen(self, *, context: ts.Context | None = None, **overrides) -> WorldArra **overrides, } return WorldArray( - self.path, context=context if context is not None else self.context, **opts + self._where, context=context if context is not None else self.context, **opts ) def __reduce__(self): """A handle pickles by its arguments and its context's recipe, and reopens in the receiving process (Behaviour 11): a worker opens its own store, cheaply, on a shared context when the handle had one, else on a context rebuilt from the same spec, so a - capped context stays capped.""" + capped context stays capped. A local path goes as resolved at open, so a worker that + changed directory still finds the store.""" opts = { **self._opts, "padding": self.padding, @@ -439,7 +459,7 @@ def __reduce__(self): "retries": self.retries, "transform": self.transform, } - return (_reopen, (self.path, opts, recipe_of(self.context))) + return (_reopen, (self._where, opts, recipe_of(self.context))) # --- index space ------------------------------------------------------------ @@ -520,7 +540,9 @@ def write_index( or the array's end), so writers in different processes need no coordination; a unit written whole is one put. ``partial=True`` lifts the check and does tensorstore's read-modify-write of the touched units, unsafe while anything else writes the same - unit. No cast, no padding, no resampling; the write commits before returning.""" + unit. ``data`` may be a scalar, which fills the box: it is converted to the array's + dtype only when the value survives (``0`` into ``uint8``, not ``300`` or ``0.5``). + No other cast, no padding, no resampling; the write commits before returning.""" if not self.writable: raise ValueError( f"{self.path} was opened read-only; open it with mode='r+' or create it" @@ -535,9 +557,11 @@ def write_index( if len(begin) != len(shape) or len(end) != len(shape): raise ValueError(f"{self.path}: begin and end need one entry per axis of {shape}") wanted = tuple(e - b for b, e in zip(begin, end)) - if data.shape != wanted: + if data.ndim == 0: + data = self._scalar(data) + elif data.shape != wanted: raise ValueError(f"{self.path}: data has shape {data.shape}; the box is {wanted}") - if data.dtype != self.dtype: + elif data.dtype != self.dtype: raise ValueError( f"{self.path}: data is {data.dtype}, the array {self.dtype}; nothing is cast on write" ) @@ -555,6 +579,22 @@ def write_index( ) self._store[tuple(slice(b, e) for b, e in zip(begin, end))].write(data).result() + def _scalar(self, value: np.ndarray) -> np.ndarray: + """A 0-d ``value`` in the array's dtype, or ``ValueError`` when converting changes it: + an integer or bool array takes only whole numbers in its range, a float array any + number that does not overflow it (rounded to its precision).""" + dtype = self.dtype + with np.errstate(invalid="ignore", over="ignore"), warnings.catch_warnings(): + warnings.simplefilter("ignore") + out = value.astype(dtype) + if dtype.kind in "biu": + kept = bool(out == value) + else: + kept = bool(np.isfinite(out) or not np.isfinite(value)) + if not kept: + raise ValueError(f"{self.path}: {value.item()!r} does not fit {dtype}") + return out + def write(self, box, data: np.ndarray, partial: bool = False) -> None: """Write ``data`` over a world box: the voxels Behaviour 2's centre rule selects, then ``write_index``. A box reaching outside the array raises; nothing is padded or @@ -880,6 +920,57 @@ def read_index(b, e, pad): data[~inside] = padding return _squeeze((data, inside) if mask else data, drop) + def read_onto( + self, + grid, + begin: Sequence[int] | None = None, + end: Sequence[int] | None = None, + padding: Any = None, + method: str | None = None, + mask: bool = False, + retries: Retry | None = None, + transform: Transform | None = None, + post_transform: Transform | None = None, + select: dict[str, int] | None = None, + ) -> np.ndarray | tuple[np.ndarray, np.ndarray]: + """Voxels ``[begin, end)`` of another grid (a ``Grid``), read from this array + (Behaviour 20): ``begin`` and ``end`` index the grid's spatial axes, all of it when + None, and the result has exactly that shape there; other axes are read as ``read`` + reads them. At the array's own voxel size nothing is resampled, however the grids are + offset: each grid voxel takes the stored voxel whose centre lies in it. Otherwise the + array is resampled onto the grid by ``method``, anchored at the grid's corner. The + grid's units are converted to ``units``; its spatial axes must be named as this + array's are.""" + sp, mine = grid.spatial, self._grid.spatial + names = tuple(grid.axes[i] for i in sp) + if names != tuple(self._grid.axes[i] for i in mine): + raise ValueError( + f"{self.path}: the grid's spatial axes are {names}; the array's " + f"{tuple(self._grid.axes[i] for i in mine)}" + ) + vs, tr = [], [] + for i, j in zip(sp, mine): + have, want = grid.units[i], self._grid.units[j] + f = 1.0 if have == want else U.factor(have, want) + vs.append(grid.voxel_size[i] * f) + tr.append(grid.translation[i] * f) + begin = (0,) * len(sp) if begin is None else tuple(int(b) for b in begin) + end = tuple(grid.shape[i] for i in sp) if end is None else tuple(int(e) for e in end) + if len(begin) != len(sp) or len(end) != len(sp): + raise ValueError(f"begin and end need one entry per spatial axis of the grid {names}") + return self.read( + Box.from_index(begin, end, vs, tr), + voxel_size=vs, + anchor=tuple(t - v / 2 for t, v in zip(tr, vs)), + padding=padding, + method=method, + mask=mask, + retries=retries, + transform=transform, + post_transform=post_transform, + select=select, + ) + def _squeeze(out, drop: tuple[int, ...]): if not drop: @@ -1123,8 +1214,8 @@ def create( 0.5) following the group's format (Behaviours 11 and 12). ``chunks`` is required. Nothing is ever deleted or overwritten: with ``exists="error"`` anything at ``path`` raises ``FileExistsError``; with ``"adopt"`` an identical array (shape, dtype, chunks, - shards, placement) is opened for writing and a different one raises ``ValueError`` - naming every difference. Returns a writable ``WorldArray``.""" + shards, placement, and every key of ``attrs`` already holding the value given) is opened + for writing and a different one raises ``ValueError`` naming every difference. Returns a writable ``WorldArray``.""" if grid is not None: # a Grid planned the output (Behaviour 18): its facts are the array's, chunks and # shards overridable; the placement arguments must not also be given @@ -1165,6 +1256,7 @@ def adopt(normalized: str, fmt: str) -> WorldArray: shards=shards, voxel_size=vs, translation=tr, + attrs=attrs, ) if diffs: raise ValueError( diff --git a/src/worldarray/write.py b/src/worldarray/write.py index d3e5884..f7fa190 100644 --- a/src/worldarray/write.py +++ b/src/worldarray/write.py @@ -803,9 +803,12 @@ def differences( shards: Sequence[int] | None, voxel_size: Sequence[float], translation: Sequence[float], + attrs: dict | None = None, ) -> list[str]: """How an existing array's ``ArrayMeta`` differs from what a caller wanted to create: - one line per field, empty when identical (placement within ``RTOL``).""" + one line per field, empty when identical (placement within ``RTOL``). Of the array's + attributes only the keys of ``attrs`` are compared, as JSON stores them; every other key + is someone else's.""" out = [] if tuple(meta.shape) != tuple(int(n) for n in shape): out.append(f"shape: found {tuple(meta.shape)}, wanted {tuple(shape)}") @@ -826,4 +829,10 @@ def differences( out.append( f"{what}: found {tuple(float(x) for x in f)}, wanted {tuple(float(x) for x in w)}" ) + found_attrs = meta.attrs or {} + for key, wanted in json.loads(json.dumps(attrs or {})).items(): + if key not in found_attrs: + out.append(f"attrs[{key!r}]: missing, wanted {wanted!r}") + elif found_attrs[key] != wanted: + out.append(f"attrs[{key!r}]: found {found_attrs[key]!r}, wanted {wanted!r}") return out diff --git a/tests/test_read.py b/tests/test_read.py index 91a6b2b..b27453f 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -554,6 +554,12 @@ def test_relabel_rescales_everything_by_the_ratio(tmp_path): resampled = wa.open(str(group / "s0"), voxel_size=(8, 8, 8), on_mismatch="resample") assert resampled.source_voxel_size == (4.0, 4.0, 4.0) assert resampled.voxel_size == (8.0, 8.0, 8.0) + for v in ( + vol, + resampled, + ): # where the level is, beside what it is (cellmap-flow's stored_placement) + assert v.source_translation == (10.0, 10.0, 10.0) + assert wa.open(str(group / "s0"), units="micrometer").source_translation == (0.01, 0.01, 0.01) def test_a_resampled_box_is_the_target_voxels_shape_counts(tmp_path): diff --git a/tests/test_resample.py b/tests/test_resample.py index 5e2c3b0..46e992e 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -139,3 +139,45 @@ def test_partial_block_mean_and_edge_fill(tmp_path): ((0, 0, 0), (48, 16, 16)), voxel_size=(8, 8, 8), method="nearest", padding="edge" ) assert list(near[:, 0, 0]) == pytest.approx([1, 3, 5, 6, 6, 6]) + + +def test_read_onto_reads_another_grids_voxels(tmp_path): + """cellmap-analyze's per-operation grid (Behaviour 20): voxels [begin, end) of a grid, + exactly that many, from any array; at the array's own voxel size nothing is resampled, + however far under a voxel the grids are offset.""" + path, data = _ramp(tmp_path, "float32") + vol = wa.open(path, method="linear") + near = wa.Grid((8, 4, 4), (4, 4, 4), (3, 3, 3)) # corner 1 nm past the array's + np.testing.assert_array_equal(vol.read_onto(near), data) # stored values, not interpolated + far = wa.Grid( + (8, 4, 4), (4, 4, 4), (5, 2, 2) + ) # 3 nm past on z: each grid voxel holds the next centre + out = vol.read_onto(far) + assert out.shape == (8, 4, 4) + np.testing.assert_array_equal(out[:7], data[1:]) + assert np.all(out[7] == 0) # the last grid voxel holds no stored centre: the fill + finer = wa.Grid((16, 8, 8), (2, 2, 2), (-1, -1, -1), chunks=(4, 4, 4)) # corner -2 + expect = vol.read(finer.box, voxel_size=(2, 2, 2), anchor=finer.corner) + np.testing.assert_array_equal(vol.read_onto(finer), expect) + halves = np.concatenate( + [vol.read_onto(finer, (0, 0, 0), (6, 8, 8)), vol.read_onto(finer, (6, 0, 0), (16, 8, 8))] + ) + np.testing.assert_array_equal(halves, expect) # adjoining reads agree with one read + in_um = wa.Grid((16, 8, 8), (0.002,) * 3, (-0.001,) * 3, units="micrometer") + np.testing.assert_array_equal(vol.read_onto(in_um), expect) + # the grid's voxels -1 and 0 are centred at -3 and -1 nm, before the data: fill, masked + out, inside = vol.read_onto(finer, (-1, 1, 1), (3, 3, 3), mask=True) + assert out.shape == (4, 2, 2) and not inside[:2].any() and inside[2:].all() + with pytest.raises(ValueError, match="spatial axes"): + vol.read_onto(wa.Grid((4, 4), (4, 4), (2, 2), axes=("y", "x"))) + + +def test_read_onto_its_own_grid_is_the_stored_array(tmp_path): + data = np.arange(7 * 6 * 5, dtype="uint16").reshape(7, 6, 5) + group = tmp_path / "odd.zarr" + write_zarr2_group(group, ome_multiscales("zyx", [("s0", (3.54, 4, 5.3), (1.0, -2.3, 7.7))])) + write_zarr2(group / "s0", data, chunks=(3, 3, 3)) + vol = wa.open(str(group / "s0")) + np.testing.assert_array_equal(vol.read_onto(wa.Grid.from_meta(vol.grid)), data) + twice = wa.Grid.from_meta(vol.grid, voxel_size=(1.77, 2, 2.65)) + assert vol.read_onto(twice).shape == twice.shape == (14, 12, 10) diff --git a/tests/test_write.py b/tests/test_write.py index ed70397..daff2cb 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -272,6 +272,19 @@ def test_create_never_overwrites_and_adopts_only_an_identical_array(tmp_path): wa.create(path, *args[:3], (3, 2, 2), chunks=(4, 4, 4), exists="adopt") +def test_adopt_checks_the_attributes_it_was_given(tmp_path): + """cellmap-flow's _refuse_another_region: funlib's prepare_ds kept an old offset silently.""" + path = str(tmp_path / "vol.zarr/s0") + args = ((8, 8, 8), "uint8", (4, 4, 4), (2, 2, 2)) + wa.create(path, *args, chunks=(4, 4, 4), attrs={"offset": (0, 0, 0), "model": "a"}) + again = wa.create(path, *args, chunks=(4, 4, 4), attrs={"offset": [0.0, 0, 0]}, exists="adopt") + assert again.meta.attrs["model"] == "a" # keys not given are not compared + with pytest.raises(ValueError) as e: + wa.create(path, *args, chunks=(4, 4, 4), attrs={"model": "b", "run": 2}, exists="adopt") + assert "attrs['model']: found 'a', wanted 'b'" in str(e.value) + assert "attrs['run']: missing, wanted 2" in str(e.value) + + def test_create_keeps_everyone_elses_attributes(tmp_path): group = tmp_path / "shared.zarr" write_zarr3_group( @@ -720,6 +733,44 @@ def test_ensure_group_entry_lists_an_array_the_package_did_not_create(tmp_path, ) +def test_a_pickled_relative_path_reopens_after_a_chdir(tmp_path, monkeypatch): + """cellmap-analyze's dask workers change directory before they unpickle.""" + monkeypatch.chdir(tmp_path) + wa.create("b.zarr/seg/s0", (8, 8, 8), "uint8", (4, 4, 4), (2, 2, 2), chunks=(4, 4, 4)) + vol = wa.open("b.zarr/seg/s0", mode="r+") + vol.write_index((0, 0, 0), (4, 4, 4), np.full((4, 4, 4), 9, np.uint8)) + blob = pickle.dumps(vol) + (tmp_path / "elsewhere").mkdir() + monkeypatch.chdir(tmp_path / "elsewhere") + again = pickle.loads(blob) + assert again.path == str(tmp_path / "b.zarr/seg/s0") + assert int(again.read_index((0, 0, 0), (1, 1, 1))[0, 0, 0]) == 9 + assert vol.reopen().level_path == again.level_path + + +def test_a_scalar_fills_the_box(tmp_path): + vol = wa.create( + str(tmp_path / "vol.zarr/s0"), (8, 8, 8), "uint8", (1, 1, 1), (0.5,) * 3, chunks=(4, 4, 4) + ) + vol.write_index((0, 0, 0), (4, 8, 8), 7) + vol.write(((4, 0, 0), (8, 4, 4)), np.uint8(3)) + vol.write_chunk((1, 1, 1), 5) + got = vol.read_index((0, 0, 0), (8, 8, 8)) + assert np.all(got[:4] == 7) and np.all(got[4:, :4, :4] == 3) and np.all(got[4:, 4:, 4:] == 5) + for bad in (300, -1, 0.5, float("nan"), True + 255): + with pytest.raises(ValueError, match="does not fit uint8"): + vol.write_index((0, 0, 0), (4, 4, 4), bad) + vol.write_index((0, 0, 0), (4, 4, 4), 0) # the fill over a whole unit leaves it absent + assert (0, 0, 0) not in vol.list_chunks() and (0, 1, 0) in vol.list_chunks() + f = wa.create( + str(tmp_path / "f.zarr/s0"), (4,), "float32", (1,), (0.5,), axes=("x",), chunks=(4,) + ) + f.write_index((0,), (4,), 0.1) # rounded to the array's precision + assert np.all(f.read_index((0,), (4,)) == np.float32(0.1)) + with pytest.raises(ValueError, match="does not fit float32"): + f.write_index((0,), (4,), 1e300) + + def test_a_pickled_handle_keeps_its_context(tmp_path): path = str(tmp_path / "vol.zarr/s0") wa.create(path, (8, 8, 8), "uint16", (4, 4, 4), (2, 2, 2), chunks=(4, 4, 4)) diff --git a/uv.lock b/uv.lock index 9b6fbf0..1780bd0 100644 --- a/uv.lock +++ b/uv.lock @@ -1247,7 +1247,7 @@ wheels = [ [[package]] name = "worldarray" -version = "0.3.0" +version = "0.4.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },